Add support for pipdeptree JSON output in pip-dependency-graph.json (l…

…ibrariesio#593)

* Add support for pipdeptree JSON support via pip-depenedency-graph.json

* 8.8.1

lib/bibliothecary/parsers/pypi.rb

@@ -9,8 +9,8 @@ class Pypi
       # Optional Group 2 is [extras].
       # Capture Group 3 is Version
       REQUIRE_REGEXP = /([a-zA-Z0-9]+[a-zA-Z0-9\-_\.]+)(?:\[.*?\])*([><=\w\.,]+)?/
-
       REQUIREMENTS_REGEXP = /^#{REQUIRE_REGEXP}/
+
       MANIFEST_REGEXP = /.*require[^\/]*(\/)?[^\/]*\.(txt|pip|in)$/
       # TODO: can this be a more specific regexp so it doesn't match something like ".yarn/cache/create-require-npm-1.0.0.zip"?
       PIP_COMPILE_REGEXP = /.*require.*$/
@@ -45,6 +45,10 @@ def self.mapping
             kind: "lockfile",
             parser: :parse_requirements_txt,
           },
+          match_filename("pip-dependency-graph.json") => { # Exported from pipdeptree --json
+            kind: "lockfile",
+            parser: :parse_dependency_tree_json,
+          },
           match_filename("setup.py") => {
             kind: "manifest",
             parser: :parse_setup_py,
@@ -226,6 +230,18 @@ def self.parse_setup_py(file_contents, options: {}) # rubocop:disable Lint/Unuse
       # should be treated as.
       NoEggSpecified = Class.new(ArgumentError)
 
+      def self.parse_dependency_tree_json(file_contents, options: {})
+        JSON.parse(file_contents)
+          .map do |pkg|
+            {
+                name: pkg.dig("package", "package_name"),
+                requirement: pkg.dig("package", "installed_version"),
+                type: "runtime",
+              }
+          end
+          .uniq
+      end
+
       # Parses a requirements.txt file, following the
       # https://pip.pypa.io/en/stable/cli/pip_install/#requirement-specifiers
       # and https://pip.pypa.io/en/stable/topics/vcs-support/#git.
@@ -252,18 +268,16 @@ def self.parse_requirements_txt(file_contents, options: {}) # rubocop:disable Li
             deps << result.merge(
               type: type
             )
-          else
-            match = line.delete(" ").match(REQUIREMENTS_REGEXP)
-            next unless match
-
+          elsif (match = line.delete(" ").match(REQUIREMENTS_REGEXP))
             deps << {
               name: match[1],
               requirement: match[-1] || "*",
               type: type,
             }
           end
         end
-        deps
+
+        deps.uniq
       end
 
       def self.parse_requirements_txt_url(url)

lib/bibliothecary/version.rb

@@ -1,3 +1,3 @@
 module Bibliothecary
-  VERSION = "8.8.0"
+  VERSION = "8.8.1"
 end

spec/fixtures/pip-dependency-graph.json

@@ -0,0 +1,244 @@
+[
+  {
+    "package": {
+      "key": "aiohttp",
+      "package_name": "aiohttp",
+      "installed_version": "3.9.5"
+    },
+    "dependencies": [
+      {
+        "key": "aiosignal",
+        "package_name": "aiosignal",
+        "installed_version": "1.3.1",
+        "required_version": ">=1.1.2"
+      },
+      {
+        "key": "async-timeout",
+        "package_name": "async-timeout",
+        "installed_version": "4.0.3",
+        "required_version": ">=4.0,<5.0"
+      },
+      {
+        "key": "attrs",
+        "package_name": "attrs",
+        "installed_version": "23.2.0",
+        "required_version": ">=17.3.0"
+      },
+      {
+        "key": "frozenlist",
+        "package_name": "frozenlist",
+        "installed_version": "1.4.1",
+        "required_version": ">=1.1.1"
+      },
+      {
+        "key": "multidict",
+        "package_name": "multidict",
+        "installed_version": "6.0.5",
+        "required_version": ">=4.5,<7.0"
+      },
+      {
+        "key": "yarl",
+        "package_name": "yarl",
+        "installed_version": "1.9.4",
+        "required_version": ">=1.0,<2.0"
+      }
+    ]
+  },
+  {
+    "package": {
+      "key": "aiosignal",
+      "package_name": "aiosignal",
+      "installed_version": "1.3.1"
+    },
+    "dependencies": [
+      {
+        "key": "frozenlist",
+        "package_name": "frozenlist",
+        "installed_version": "1.4.1",
+        "required_version": ">=1.1.0"
+      }
+    ]
+  },
+  {
+    "package": {
+      "key": "async-timeout",
+      "package_name": "async-timeout",
+      "installed_version": "4.0.3"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "attrs",
+      "package_name": "attrs",
+      "installed_version": "23.2.0"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "black",
+      "package_name": "black",
+      "installed_version": "23.12.0"
+    },
+    "dependencies": [
+      {
+        "key": "aiohttp",
+        "package_name": "aiohttp",
+        "installed_version": "3.9.5",
+        "required_version": ">=3.7.4"
+      },
+      {
+        "key": "click",
+        "package_name": "click",
+        "installed_version": "8.1.7",
+        "required_version": ">=8.0.0"
+      },
+      {
+        "key": "mypy-extensions",
+        "package_name": "mypy-extensions",
+        "installed_version": "1.0.0",
+        "required_version": ">=0.4.3"
+      },
+      {
+        "key": "packaging",
+        "package_name": "packaging",
+        "installed_version": "24.0",
+        "required_version": ">=22.0"
+      },
+      {
+        "key": "pathspec",
+        "package_name": "pathspec",
+        "installed_version": "0.12.1",
+        "required_version": ">=0.9.0"
+      },
+      {
+        "key": "platformdirs",
+        "package_name": "platformdirs",
+        "installed_version": "4.2.2",
+        "required_version": ">=2"
+      },
+      {
+        "key": "tomli",
+        "package_name": "tomli",
+        "installed_version": "2.0.1",
+        "required_version": ">=1.1.0"
+      },
+      {
+        "key": "typing-extensions",
+        "package_name": "typing_extensions",
+        "installed_version": "4.12.0",
+        "required_version": ">=4.0.1"
+      }
+    ]
+  },
+  {
+    "package": {
+      "key": "click",
+      "package_name": "click",
+      "installed_version": "8.1.7"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "frozenlist",
+      "package_name": "frozenlist",
+      "installed_version": "1.4.1"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "idna",
+      "package_name": "idna",
+      "installed_version": "3.7"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "multidict",
+      "package_name": "multidict",
+      "installed_version": "6.0.5"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "mypy-extensions",
+      "package_name": "mypy-extensions",
+      "installed_version": "1.0.0"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "packaging",
+      "package_name": "packaging",
+      "installed_version": "24.0"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "pathspec",
+      "package_name": "pathspec",
+      "installed_version": "0.12.1"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "platformdirs",
+      "package_name": "platformdirs",
+      "installed_version": "4.2.2"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "termcolor",
+      "package_name": "termcolor",
+      "installed_version": "2.4.0"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "tomli",
+      "package_name": "tomli",
+      "installed_version": "2.0.1"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "typing-extensions",
+      "package_name": "typing_extensions",
+      "installed_version": "4.12.0"
+    },
+    "dependencies": []
+  },
+  {
+    "package": {
+      "key": "yarl",
+      "package_name": "yarl",
+      "installed_version": "1.9.4"
+    },
+    "dependencies": [
+      {
+        "key": "idna",
+        "package_name": "idna",
+        "installed_version": "3.7",
+        "required_version": ">=2.0"
+      },
+      {
+        "key": "multidict",
+        "package_name": "multidict",
+        "installed_version": "6.0.5",
+        "required_version": ">=4.0"
+      }
+    ]
+  }
+]

spec/parsers/pypi_spec.rb

@@ -204,6 +204,69 @@
     })
   end
 
+  it "parses dependencies from pip-dependency-graph.json" do
+    expect(described_class.analyse_contents("pip-dependency-graph.json", load_fixture("pip-dependency-graph.json"))).to eq({
+      platform: "pypi",
+      path: "pip-dependency-graph.json",
+      dependencies: [
+        { name: "aiohttp", requirement: "3.9.5", type: "runtime" },
+        { name: "aiosignal", requirement: "1.3.1", type: "runtime" },
+        { name: "async-timeout", requirement: "4.0.3", type: "runtime" },
+        { name: "attrs", requirement: "23.2.0", type: "runtime" },
+        { name: "black", requirement: "23.12.0", type: "runtime" },
+        { name: "click", requirement: "8.1.7", type: "runtime" },
+        { name: "frozenlist", requirement: "1.4.1", type: "runtime" },
+        { name: "idna", requirement: "3.7", type: "runtime" },
+        { name: "multidict", requirement: "6.0.5", type: "runtime" },
+        { name: "mypy-extensions", requirement: "1.0.0", type: "runtime" },
+        { name: "packaging", requirement: "24.0", type: "runtime" },
+        { name: "pathspec", requirement: "0.12.1", type: "runtime" },
+        { name: "platformdirs", requirement: "4.2.2", type: "runtime" },
+        { name: "termcolor", requirement: "2.4.0", type: "runtime" },
+        { name: "tomli", requirement: "2.0.1", type: "runtime" },
+        { name: "typing_extensions", requirement: "4.12.0", type: "runtime" },
+        { name: "yarl", requirement: "1.9.4", type: "runtime" },
+      ],
+      kind: "lockfile",
+      success: true,
+    })
+  end
+
+
+  it "handles duplicate dependencies from pip-dependency-graph.json" do
+    # It doesn't seem possible that pipdeptree would output duplicate
+    # dependencies, but this ensures we catch it in case that is possible.
+    lockfile = <<-JSON
+    [
+      {
+        "package": {
+          "key": "aiohttp",
+          "package_name": "aiohttp",
+          "installed_version": "3.9.5"
+        },
+        "dependencies": []
+      },
+      {
+        "package": {
+          "key": "aiohttp",
+          "package_name": "aiohttp",
+          "installed_version": "3.9.5"
+        },
+        "dependencies": []
+      }
+    ]    
+    JSON
+    expect(described_class.analyse_contents("pip-dependency-graph.json", lockfile)).to eq({
+      platform: "pypi",
+      path: "pip-dependency-graph.json",
+      dependencies: [
+        { name: "aiohttp", requirement: "3.9.5", type: "runtime" },
+      ],
+      kind: "lockfile",
+      success: true,
+    })
+  end
+
   it "parses dependencies from requirements.frozen" do
     expect(described_class.analyse_contents("requirements.frozen", load_fixture("requirements.frozen"))).to eq({
       platform: "pypi",