Make output verbose

.gitignore

@@ -24,7 +24,6 @@ tramp
 # Org-mode
 .org-id-locations
 *_archive
-todo.org
 
 # flymake-mode
 *_flymake.*
@@ -60,3 +59,6 @@ flycheck_*.el
 
 # network security
 /network-security.data
+
+# misc
+scratch.md

CHANGELOG.md

@@ -5,6 +5,22 @@ This change log follows the conventions of [keepachangelog.com](http://keepachan
 
 ## [Unreleased]
 
+## [0.38.0] - 2022-11-27
+
+Release **breaks backward compatibility** if you use the tool as a
+library in your Clojure code (see `Changed` section).
+
+### Added
+- CLI option `--verbose` (or `-v` for short) to make output
+  verbose. Options adds `Misc` column for error messages to the
+  report.
+  ([#105](https://github.com/pilosus/pip-license-checker/issues/105))
+
+### Changed
+- Massive code refactoring: multiple functions renamed, a few removed,
+  some function arguments order changed
+  (see [PR 115](https://github.com/pilosus/pip-license-checker/pull/115/files)).
+
 ## [0.37.0] - 2022-11-23
 ### Changed
 - Docker base image switched over vendor-agnostic Eclipse Temurin JRE 17.
@@ -297,7 +313,8 @@ weak copyleft types.
 ### Added
 - Structure for Leiningen app project
 
-[Unreleased]: https://github.com/pilosus/pip-license-checker/compare/0.37.0...HEAD
+[Unreleased]: https://github.com/pilosus/pip-license-checker/compare/0.38.0...HEAD
+[0.38.0]: https://github.com/pilosus/pip-license-checker/compare/0.37.0...0.38.0
 [0.37.0]: https://github.com/pilosus/pip-license-checker/compare/0.36.0...0.37.0
 [0.36.0]: https://github.com/pilosus/pip-license-checker/compare/0.35.0...0.36.0
 [0.35.0]: https://github.com/pilosus/pip-license-checker/compare/0.34.0...0.35.0

README.md

@@ -95,11 +95,14 @@ docker's `-v` option:
 
 ```bash
 docker run -v `pwd`:/volume \
+    -e GITHUB_TOKEN=your-gh-token \
     -it --rm pilosus/pip-license-checker \
     java -jar app.jar --exclude 'pylint.*' \
     --requirements '/volume/requirements.txt' \
     --external '/volume/licenses.csv' \
-    --fail StrongCopyleft --fails-only
+    --fail StrongCopyleft \
+    --fails-only \
+    --verbose
 ```
 
 ### Command line tool
@@ -160,6 +163,7 @@ pip-license-checker [options]... [package]...
 Description:
   package	List of package names in format `name[specifier][version]`
 
+  -v, --verbose                           false                                   Make output verbose
   -r, --requirements REQUIREMENT_NAME     []                                      Requirement file name to read
   -x, --external FILE_NAME                []                                      File containing package names and license names
       --external-format FILE_FORMAT       csv                                     External file format: csv, cocoapods, edn, gradle

project.clj

@@ -1,12 +1,12 @@
-(defproject org.clojars.vrs/pip-license-checker "0.37.0"
+(defproject org.clojars.vrs/pip-license-checker "0.38.0-SNAPSHOT"
   :description "License compliance tool to identify dependencies license names and types: permissive, copyleft, proprietory, etc."
   :url "https://github.com/pilosus/pip-license-checker"
   :license {:name "Eclipse Public License 2.0 OR GNU GPL v2+ with Classpath exception"
             :url "https://github.com/pilosus/pip-license-checker/blob/main/LICENSE"}
   :dependencies [
                  [org.clojure/clojure "1.10.1"]
                  [org.clojure/data.csv "1.0.1"]
-                 [org.clojure/tools.cli "1.0.206"]
+                 [org.clojure/tools.cli "1.0.214"]
                  [clj-http "3.12.3"]
                  [cheshire "5.11.0"]
                  [indole "1.0.0"]

resources/requirements.txt

@@ -10,7 +10,9 @@ aiopg == 122.3.5  # no such version, error expected
 telegram-bot-framework==3.15.2  # Network Copyleft
 aio-throttle==1.7.0  # no license on PyPI, look up GitHub
 pygit2==1.6.1  # GPL with linking exception, Weak Copyleft
+pypi-project-no-license==0.1.1  # no PyPI license, not GitHub license
 
 https://example.com/package_3.0.3.dev1820+49a8884-cp34-none-win_amd64.whl
 workflow-tools>=0.5.0,<=0.6.0 ; python_version > '3.6'
 Synx  # proprietary license
+this-project-does-not-exist  # No such project

src/pip_license_checker/core.clj

@@ -18,16 +18,14 @@
   (:gen-class)
   (:require
    [clojure.set :refer [intersection]]
-   [clojure.spec.alpha :as s]
    [clojure.string :as str]
    [clojure.tools.cli :refer [parse-opts]]
    [pip-license-checker.external :as external]
    [pip-license-checker.file :as file]
    [pip-license-checker.filters :as filters]
    [pip-license-checker.license :as license]
    [pip-license-checker.pypi :as pypi]
-   [pip-license-checker.report :as report]
-   [pip-license-checker.spec :as sp]))
+   [pip-license-checker.report :as report]))
 
 (defn exit
   "Exit from the app with exit status"
@@ -37,41 +35,34 @@
    (println msg)
    (exit status)))
 
-(s/fdef get-license-type-totals
-  :args (s/coll-of ::sp/requirement-response-license)
-  :ret ::sp/license-type-totals)
-
 (defn get-license-type-totals
   "Return a frequency map of license types as keys and license types as values"
   [licenses]
-  (let [freqs (frequencies (map #(:type (:license %)) licenses))
-        ordered-freqs (into (sorted-map) freqs)]
-    ordered-freqs))
-
-(s/fdef process-requirements
-  :args (s/cat
-         :requirements ::sp/requirements-cli-arg
-         :packages ::sp/packages-cli-arg
-         :options ::sp/options-cli-arg))
-
-(defn process-requirements
-  "Print parsed requirements pretty"
+  (->> licenses
+       (map #(:type (:license %)))
+       frequencies
+       (into (sorted-map))))
+
+(defn process-deps
+  "Print parsed dependencies pretty"
   [packages requirements external options]
-  (let [fail-opt (:fail options)
+  (let [{table-headers :table-headers
+         fail-opt :fail
+         with-totals-opt :with-totals
+         totals-only-opt :totals-only} options
         with-fail (seq fail-opt)
-        with-totals-opt (:with-totals options)
-        totals-only-opt (:totals-only options)
         show-totals (or with-totals-opt totals-only-opt)
-        table-headers (:table-headers options)
-        parsed-external-licenses (external/get-parsed-requiements external options)
-        parsed-pypi-licenses (pypi/get-parsed-requiements packages requirements options)
-        parsed-licenses (concat parsed-pypi-licenses parsed-external-licenses)
-        licenses (filters/filter-parsed-requirements parsed-licenses options)
-        totals
-        (if (or show-totals with-fail)
-          (get-license-type-totals licenses)
-          nil)
-        totals-keys (into (sorted-set) (keys totals))
+
+        deps (concat
+              (pypi/get-parsed-deps packages requirements options)
+              (external/get-parsed-deps external options))
+        licenses (filters/filter-parsed-deps deps options)
+
+        totals (when (or show-totals with-fail) (get-license-type-totals licenses))
+        totals-keys (->> totals
+                         keys
+                         (into (sorted-set)))
+
         fail-types-found (intersection totals-keys fail-opt)
         fail? (seq fail-types-found)]
 
@@ -148,7 +139,8 @@
   "Rate limits must be positive integers in format REQUESTS/MILLISECONDS")
 
 (def cli-options
-  [["-r" "--requirements REQUIREMENTS_FILE" "Python pip requirement file name"
+  [["-v" "--verbose" "Make output verbose" :default false]
+   ["-r" "--requirements REQUIREMENTS_FILE" "Python pip requirement file name"
     :multi true
     :default []
     :update-fn conj
@@ -187,10 +179,6 @@
     :default (System/getenv "GITHUB_TOKEN")]
    ["-h" "--help" "Print this help message"]])
 
-(s/fdef extend-fail-opt
-  :args (s/? ::sp/options-fail)
-  :ret (s/? ::sp/options-fail))
-
 (defn extend-fail-opt
   "Try to substitute common fail option in set with specific parts it consist of"
   [fail-opts]
@@ -200,10 +188,6 @@
      #{license/type-copyleft-all})
     fail-opts))
 
-(s/fdef post-process-options
-  :args (s/? ::sp/options-cli-arg)
-  :ret (s/? ::sp/options-cli-arg))
-
 (defn post-process-options
   "Update option map"
   [options]
@@ -213,15 +197,6 @@
         updated-opts (assoc opts' :fail fail-opt-exteded)]
     updated-opts))
 
-(s/fdef validate-args
-  :args (s/cat :args (s/coll-of string?))
-  :ret (s/cat
-        :exit-message (s/? string?)
-        :ok? (s/? boolean?)
-        :requirements (s/? ::sp/requirements-cli-arg)
-        :packages (s/? ::sp/packages-cli-arg)
-        :options (s/? ::sp/options-cli-arg)))
-
 (defn validate-args
   "Parse and validate CLI arguments for entrypoint"
   [args]
@@ -246,4 +221,4 @@
   (let [{:keys [packages requirements external options exit-message ok?]} (validate-args args)]
     (if exit-message
       (exit (if ok? 0 1) exit-message)
-      (process-requirements packages requirements external options))))
+      (process-deps packages requirements external options))))

src/pip_license_checker/data.clj

@@ -0,0 +1,46 @@
+;; Copyright © 2020-2022 Vitaly Samigullin
+;;
+;; This program and the accompanying materials are made available under the
+;; terms of the Eclipse Public License 2.0 which is available at
+;; http://www.eclipse.org/legal/epl-2.0.
+;;
+;; This Source Code may also be made available under the following Secondary
+;; Licenses when the conditions for such availability set forth in the Eclipse
+;; Public License, v. 2.0 are satisfied: GNU General Public License as published by
+;; the Free Software Foundation, either version 2 of the License, or (at your
+;; option) any later version, with the GNU Classpath Exception which is available
+;; at https://www.gnu.org/software/classpath/license.html.
+;;
+;; SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+
+(ns pip-license-checker.data
+  "Data structures and helper functions for them"
+  (:gen-class))
+
+(defrecord License
+           [name  ;; nilable String
+            type  ;; nilable String
+            error ;; nilable String
+            ])
+
+(defrecord Requirement
+           [name       ;; nilable String
+            version    ;; nilable String
+            specifiers ;; nilable vector of vectors of format [op, version]; nil for non-Python deps
+            ])
+
+(defrecord PyPiProject
+           ;; PyPI project as represented on https://pypi.org/project/<project-name>
+           [status        ;; keyword
+            requirement   ;; Requirement rec
+            api-response  ;; nilable parsed JSON
+            license       ;; License rec
+            error         ;; nilable String
+            ])
+
+(defrecord Dependency
+           ;; General representation of dependency - PyPI project or external dep
+           [requirement ;; Requirement rec
+            license     ;; License rec
+            error       ;; nilable String
+            ])

src/pip_license_checker/exception.clj

@@ -0,0 +1,33 @@
+;; Copyright © 2020-2022 Vitaly Samigullin
+;;
+;; This program and the accompanying materials are made available under the
+;; terms of the Eclipse Public License 2.0 which is available at
+;; http://www.eclipse.org/legal/epl-2.0.
+;;
+;; This Source Code may also be made available under the following Secondary
+;; Licenses when the conditions for such availability set forth in the Eclipse
+;; Public License, v. 2.0 are satisfied: GNU General Public License as published by
+;; the Free Software Foundation, either version 2 of the License, or (at your
+;; option) any later version, with the GNU Classpath Exception which is available
+;; at https://www.gnu.org/software/classpath/license.html.
+;;
+;; SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+
+(ns pip-license-checker.exception
+  "Error handling"
+  (:gen-class)
+  (:require
+   [clojure.string :as str]))
+
+(defn get-ex-info
+  "Generate a human-readable exception message"
+  [logger ex]
+  (let [name (-> ex .getClass .getSimpleName)
+        message (ex-message ex)]
+    (format "[%s] %s: %s" logger name message)))
+
+(defn join-ex-info
+  "Join exception messages"
+  [& args]
+  (let [ex (str/join "; " (concat (filter some? args)))]
+    (if (str/blank? ex) nil ex)))