Migrate xen importer

vulnerabilities/importers/__init__.py

@@ -31,6 +31,7 @@
 from vulnerabilities.importers import retiredotnet
 from vulnerabilities.importers import suse_scores
 from vulnerabilities.importers import ubuntu
+from vulnerabilities.importers import xen
 
 IMPORTERS_REGISTRY = [
     nginx.NginxImporter,
@@ -57,6 +58,7 @@
     suse_scores.SUSESeverityScoreImporter,
     elixir_security.ElixirSecurityImporter,
     apache_tomcat.ApacheTomcatImporter,
+    xen.XenImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/xen.py

@@ -7,66 +7,77 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import json
-
-import requests
-
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
-from vulnerabilities.importer import Reference
-from vulnerabilities.utils import create_etag
+from vulnerabilities.references import XsaReference
+from vulnerabilities.utils import fetch_response
 from vulnerabilities.utils import is_cve
 
 
 class XenImporter(Importer):
-    # CONFIG_CLASS = XenDBConfiguration
-
-    def updated_advisories(self):
-        advisories = []
-        if create_etag(data_src=self, url=self.config.db_url, etag_key="etag"):
-            advisories.extend(self.to_advisories(fetch(self.config.db_url)))
-
-        return self.batch_advisories(advisories)
-
-    def create_etag(self, url):
-        etag = requests.head(url).headers.get("etag")
-        if not etag:
-            return True
-
-        elif url in self.config.etags:
-            if self.config.etags[url] == etag:
-                return False
-
-        self.config.etags[url] = etag
-        return True
-
-    @staticmethod
-    def to_advisories(xen_db):
-        advisories = []
-        for xsa in xen_db[0]["xsas"]:
-            reference = get_xen_references(xsa["xsa"])
-            title = xsa.get("title", [""])
-            for cve in xsa.get("cve", [""]):
-                if not is_cve(cve):
-                    cve = ""
-
-                advisories.append(
-                    AdvisoryData(
-                        vulnerability_id=cve,
-                        summary=title,
-                        references=[reference],
-                    )
-                )
-        return advisories
-
-
-def get_xen_references(xsa_id):
-    return Reference(
-        reference_id="XSA-" + xsa_id,
-        url="https://xenbits.xen.org/xsa/advisory-{}.html".format(xsa_id),
-    )
-
 
-def fetch(url):
-    response = requests.get(url).content
-    return json.loads(response)
+    url = "https://xenbits.xen.org/xsa/xsa.json"
+    spdx_license_expression = "LicenseRef-scancode-other-permissive"
+    notice = """
+    From: George Dunlap <george.dunlap@cloud.com>
+    Date: Wed, Jan 25, 2023 at 4:57 PM
+    Subject: Re: Usage of Xen Security Data in VulnerableCode
+    To: Tushar Goel <tushar.goel.dav@gmail.com>
+    Cc: Andrew Cooper <Andrew.Cooper3@citrix.com>, xen-devel@lists.xenproject.org <xen-devel@lists.xenproject.org>, Xen Security <security@xen.org>, Philippe Ombredanne <pombredanne@nexb.com>, <jmhoran@nexb.com>
+
+    On Thu, Jan 19, 2023 at 1:10 PM Tushar Goel <tushar.goel.dav@gmail.com> wrote:
+    >
+    > Hi Andrew,
+    >
+    > > Maybe we want to make it CC-BY-4 to require people to reference back to
+    > > the canonical upstream ?
+    > Thanks for your response, can we have a more declarative statement on
+    > the license from your end
+    > and also can you please provide your acknowledgement over the usage of
+    > Xen security data in vulnerablecode.
+
+
+    Hey Tushar,
+    Informally, the Xen Project Security Team is happy for you to include the data from xsa.json in your open-source vulnerability database. As a courtesy we'd request that it be documented where the information came from. (I think if the data includes links to then advisories on our website, that will suffice.)
+    Formally, we're not copyright lawyers; but we don't think there's anything copyright-able in the xsa.json: There is no editorial or creative control in the generation of that file; it's just a collection of facts which you could re-generate by scanning all the advisories. (In fact that's exactly how the file is created; i.e., the collection of advisory texts is our "source of truth".)
+    We do have "Officially license all advisory text as CC-BY-4" on our to-do list; if you'd be more comfortable with an official license for xsa.json as well, we can add that to the list.
+
+     -George
+    """
+
+    def advisory_data(self):
+        data = fetch_response(self.url).json()
+        # The data looks like this
+        # [
+        #  {
+        #   "xsas": [
+        #     {
+        #       "cve": [
+        #         "CVE-2012-5510"
+        #       ],
+        #       "title": "XSA-1: Xen security advisory",
+        #       }
+        #     ]
+        #   }
+        # ]
+        if not data:
+            return []
+        xsas = data[0]["xsas"]
+        for xsa in xsas:
+            yield from self.to_advisories(xsa)
+
+    def to_advisories(self, xsa):
+        xsa_id = xsa.get("xsa")
+        references = []
+        if xsa_id:
+            references.append(XsaReference.from_number(number=xsa_id))
+        title = xsa.get("title")
+        for cve in xsa.get("cve") or []:
+            # TODO: https://github.com/nexB/vulnerablecode/issues/981
+            if not is_cve(cve):
+                continue
+            yield AdvisoryData(
+                aliases=[cve],
+                summary=title,
+                references=references,
+            )

vulnerabilities/references.py

@@ -28,6 +28,16 @@ def from_id(cls, xsa_id):
             url=f"https://xenbits.xen.org/xsa/advisory-{numid}.html",
         )
 
+    @classmethod
+    def from_number(cls, number):
+        """
+        Return a new XsaReference from an XSA number.
+        """
+        return cls(
+            reference_id=f"XSA-{number}",
+            url=f"https://xenbits.xen.org/xsa/advisory-{number}.html",
+        )
+
 
 class ZbxReference(Reference):
     """

vulnerabilities/tests/test_data/parse-advisory-xen-expected.json

@@ -0,0 +1,18 @@
+[
+  {
+    "aliases": [
+      "CVE-2012-5510"
+    ],
+    "summary": "Grant table version switch list corruption vulnerability",
+    "affected_packages": [],
+    "references": [
+      {
+        "reference_id": "XSA-26",
+        "url": "https://xenbits.xen.org/xsa/advisory-26.html",
+        "severities": []
+      }
+    ],
+    "date_published": null,
+    "weaknesses": []
+  }
+]

vulnerabilities/tests/test_data/xen_data.json

@@ -0,0 +1,42 @@
+{
+    "cve": [
+      "CVE-2012-5510"
+    ],
+    "files": [
+      {
+        "name": "xsa26-4.1.patch",
+        "project": "xen",
+        "upstream": "Xen",
+        "versions": [
+          [
+            "4.1.x"
+          ]
+        ]
+      },
+      {
+        "name": "xsa26-4.2.patch",
+        "project": "xen",
+        "upstream": "Xen",
+        "versions": [
+          [
+            "4.2.x"
+          ]
+        ]
+      },
+      {
+        "name": "xsa26-unstable.patch",
+        "project": "xen",
+        "upstream": "Xen",
+        "versions": [
+          [
+            "xen-unstable"
+          ]
+        ]
+      }
+    ],
+    "public_time": "2012-12-03 17:51",
+    "title": "Grant table version switch list corruption vulnerability",
+    "version": 3,
+    "version_time": "2012-12-03 17:51",
+    "xsa": "26"
+}

vulnerabilities/tests/test_xen.py

@@ -0,0 +1,29 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+import os
+
+from vulnerabilities.importers.xen import XenImporter
+from vulnerabilities.tests import util_tests
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+TEST_DATA = os.path.join(
+    BASE_DIR,
+    "test_data",
+)
+
+
+def test_xen_to_advisories():
+    with open(os.path.join(TEST_DATA, "xen_data.json")) as f:
+        raw_data = json.load(f)
+    advisories = XenImporter().to_advisories(raw_data)
+    result = [data.to_dict() for data in advisories]
+    expected_file = os.path.join(TEST_DATA, f"parse-advisory-xen-expected.json")
+    util_tests.check_results_against_json(result, expected_file)