Add RubyImporter to git_importer test_git_importer_clone (#799)

Drop cvss_v2
Add ruby importer_name and Rebase
Resolve merge conflicts
Add advisory_url to ruby importer
Add a notice and the spdx_license_expression
Resolve merge conflict
Add a docstring to get_affected_packages
Add a unite test for get_affected_packages function
Remove unused variables
Fix sorted affected_package_merge
Add ruby importer and improver
Fix style test
Fix test
Rewrite affected_packages
Ruby initial config
Reference: #796

Clean imported data after import process


Fix sorted affected_package_merge
Refactor Ruby importer and improver
Add ruby importer and improver
Fix style test
Fix test
Rewrite affected_packages
Ruby initial config
Reference: #796

Signed-off-by: ziadhany <ziadhany2016@gmail.com>

vulnerabilities/importers/__init__.py

@@ -32,6 +32,7 @@
 from vulnerabilities.importers import pysec
 from vulnerabilities.importers import redhat
 from vulnerabilities.importers import retiredotnet
+from vulnerabilities.importers import ruby
 from vulnerabilities.importers import suse_scores
 from vulnerabilities.importers import ubuntu
 from vulnerabilities.importers import ubuntu_usn
@@ -67,6 +68,7 @@
     fireeye.FireyeImporter,
     apache_kafka.ApacheKafkaImporter,
     oss_fuzz.OSSFuzzImporter,
+    ruby.RubyImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/ruby.py

@@ -7,132 +7,177 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import asyncio
-from typing import List
-from typing import Set
+import logging
+from pathlib import Path
+from typing import Iterable
 
 from dateutil.parser import parse
 from packageurl import PackageURL
 from pytz import UTC
-from univers.version_range import VersionRange
-from univers.versions import SemverVersion
+from univers.version_range import GemVersionRange
 
 from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
-from vulnerabilities.package_managers import RubyVersionAPI
+from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.severity_systems import SCORING_SYSTEMS
+from vulnerabilities.utils import build_description
+from vulnerabilities.utils import get_advisory_url
 from vulnerabilities.utils import load_yaml
-from vulnerabilities.utils import nearest_patched_package
 
+logger = logging.getLogger(__name__)
 
-class RubyImporter(Importer):
-    def __enter__(self):
-        super(RubyImporter, self).__enter__()
-
-        if not getattr(self, "_added_files", None):
-            self._added_files, self._updated_files = self.file_changes(
-                recursive=True, file_ext="yml", subdir="./gems"
-            )
 
-        self.pkg_manager_api = RubyVersionAPI()
-        self.set_api(self.collect_packages())
-
-    def set_api(self, packages):
-        asyncio.run(self.pkg_manager_api.load_api(packages))
-
-    def updated_advisories(self) -> Set[AdvisoryData]:
-        files = self._updated_files.union(self._added_files)
-        advisories = []
-        for f in files:
-            processed_data = self.process_file(f)
-            if processed_data:
-                advisories.append(processed_data)
-        return self.batch_advisories(advisories)
-
-    def collect_packages(self):
-        packages = set()
-        files = self._updated_files.union(self._added_files)
-        for f in files:
-            data = load_yaml(f)
-            if data.get("gem"):
-                packages.add(data["gem"])
-
-        return packages
-
-    def process_file(self, path) -> List[AdvisoryData]:
-        record = load_yaml(path)
+class RubyImporter(Importer):
+    license_url = "https://github.com/rubysec/ruby-advisory-db/blob/master/LICENSE.txt"
+    repo_url = "git+https://github.com/rubysec/ruby-advisory-db"
+    importer_name = "Ruby Importer"
+    spdx_license_expression = "LicenseRef-scancode-public-domain-disclaimer"
+    notice = """
+    If you submit code or data to the ruby-advisory-db that is copyrighted by
+    yourself, upon submission you hereby agree to release it into the public
+    domain.
+
+    The data imported from the ruby-advisory-db have been filtered to exclude 
+    any non-public domain data from the data copyrighted by the Open 
+    Source Vulnerability Database (http://osvdb.org).
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+    """
+
+    def advisory_data(self) -> Iterable[AdvisoryData]:
+        try:
+            self.clone(self.repo_url)
+            base_path = Path(self.vcs_response.dest_dir)
+            supported_subdir = ["rubies", "gems"]
+            for subdir in supported_subdir:
+                for file_path in base_path.glob(f"{subdir}/**/*.yml"):
+                    if file_path.name.startswith("OSVDB-"):
+                        continue
+                    raw_data = load_yaml(file_path)
+                    advisory_url = get_advisory_url(
+                        file=file_path,
+                        base_path=base_path,
+                        url="https://github.com/rubysec/ruby-advisory-db/blob/master/",
+                    )
+                    yield parse_ruby_advisory(raw_data, subdir, advisory_url)
+        finally:
+            if self.vcs_response:
+                self.vcs_response.delete()
+
+
+def parse_ruby_advisory(record, schema_type, advisory_url):
+    """
+    Parse a ruby advisory file and return an AdvisoryData or None.
+    Each advisory file contains the advisory information in YAML format.
+    Schema: https://github.com/rubysec/ruby-advisory-db/tree/master/spec/schemas
+    """
+    if schema_type == "gems":
         package_name = record.get("gem")
-        if not package_name:
-            return
 
-        if "cve" in record:
-            cve_id = "CVE-{}".format(record["cve"])
+        if not package_name:
+            logger.error("Invalid package name")
         else:
-            return
-
-        publish_time = parse(record["date"]).replace(tzinfo=UTC)
-        safe_version_ranges = record.get("patched_versions", [])
-        # this case happens when the advisory contain only 'patched_versions' field
-        # and it has value None(i.e it is empty :( ).
-        if not safe_version_ranges:
-            safe_version_ranges = []
-        safe_version_ranges += record.get("unaffected_versions", [])
-        safe_version_ranges = [i for i in safe_version_ranges if i]
-
-        if not getattr(self, "pkg_manager_api", None):
-            self.pkg_manager_api = RubyVersionAPI()
-        all_vers = self.pkg_manager_api.get(package_name, until=publish_time).valid_versions
-        safe_versions, affected_versions = self.categorize_versions(all_vers, safe_version_ranges)
-
-        impacted_purls = [
-            PackageURL(
-                name=package_name,
-                type="gem",
-                version=version,
+            purl = PackageURL(type="gem", name=package_name)
+
+            return AdvisoryData(
+                aliases=get_aliases(record),
+                summary=get_summary(record),
+                affected_packages=get_affected_packages(record, purl),
+                references=get_references(record),
+                date_published=get_publish_time(record),
+                url=advisory_url,
             )
-            for version in affected_versions
-        ]
-
-        resolved_purls = [
-            PackageURL(
-                name=package_name,
-                type="gem",
-                version=version,
+
+    elif schema_type == "rubies":
+        engine = record.get("engine")  # engine enum: [jruby, rbx, ruby]
+        if not engine:
+            logger.error("Invalid engine name")
+        else:
+            purl = PackageURL(type="ruby", name=engine)
+            return AdvisoryData(
+                aliases=get_aliases(record),
+                summary=get_summary(record),
+                affected_packages=get_affected_packages(record, purl),
+                references=get_references(record),
+                date_published=get_publish_time(record),
+                url=advisory_url,
             )
-            for version in safe_versions
-        ]
 
-        references = []
-        if record.get("url"):
-            references.append(Reference(url=record.get("url")))
 
-        return AdvisoryData(
-            summary=record.get("description", ""),
-            affected_packages=nearest_patched_package(impacted_purls, resolved_purls),
-            references=references,
-            vulnerability_id=cve_id,
+def get_affected_packages(record, purl):
+    """
+    Return AffectedPackage objects one for each affected_version_range and invert the safe_version_ranges
+    ( patched_versions , unaffected_versions ) then passing the purl and the inverted safe_version_range
+    to the AffectedPackage object
+    """
+    safe_version_ranges = record.get("patched_versions", [])
+    # this case happens when the advisory contain only 'patched_versions' field
+    # and it has value None(i.e it is empty :( ).
+    if not safe_version_ranges:
+        safe_version_ranges = []
+    safe_version_ranges += record.get("unaffected_versions", [])
+    safe_version_ranges = [i for i in safe_version_ranges if i]
+
+    affected_packages = []
+    affected_version_ranges = [
+        GemVersionRange.from_native(elem).invert() for elem in safe_version_ranges
+    ]
+
+    for affected_version_range in affected_version_ranges:
+        affected_packages.append(
+            AffectedPackage(
+                package=purl,
+                affected_version_range=affected_version_range,
+            )
         )
+    return affected_packages
+
 
-    @staticmethod
-    def categorize_versions(all_versions, unaffected_version_ranges):
+def get_aliases(record) -> [str]:
+    aliases = []
+    if record.get("cve"):
+        aliases.append("CVE-{}".format(record.get("cve")))
+    if record.get("osvdb"):
+        aliases.append("OSV-{}".format(record.get("osvdb")))
+    if record.get("ghsa"):
+        aliases.append("GHSA-{}".format(record.get("ghsa")))
+    return aliases
 
-        for id, elem in enumerate(unaffected_version_ranges):
-            unaffected_version_ranges[id] = VersionRange.from_scheme_version_spec_string(
-                "semver", elem
+
+def get_references(record) -> [Reference]:
+    references = []
+    cvss_v3 = record.get("cvss_v3")
+    if record.get("url"):
+        if not cvss_v3:
+            references.append(Reference(url=record.get("url")))
+        else:
+            references.append(
+                Reference(
+                    url=record.get("url"),
+                    severities=[
+                        VulnerabilitySeverity(system=SCORING_SYSTEMS["cvssv3"], value=cvss_v3)
+                    ],
+                )
             )
+    return references
+
+
+def get_publish_time(record):
+    date = record.get("date")
+    if not date:
+        return
+    return parse(date).replace(tzinfo=UTC)
+
 
-        safe_versions = []
-        vulnerable_versions = []
-        for i in all_versions:
-            vobj = SemverVersion(i)
-            is_vulnerable = False
-            for ver_rng in unaffected_version_ranges:
-                if vobj in ver_rng:
-                    safe_versions.append(i)
-                    is_vulnerable = True
-                    break
-
-            if not is_vulnerable:
-                vulnerable_versions.append(i)
-
-        return safe_versions, vulnerable_versions
+def get_summary(record):
+    title = record.get("title") or ""
+    description = record.get("description") or ""
+    return build_description(summary=title, description=description)

vulnerabilities/improvers/__init__.py

@@ -24,6 +24,7 @@
     valid_versions.DebianOvalImprover,
     valid_versions.UbuntuOvalImprover,
     valid_versions.OSSFuzzImprover,
+    valid_versions.RubyImprover,
     vulnerability_status.VulnerabilityStatusImprover,
 ]
 

vulnerabilities/improvers/valid_versions.py

@@ -37,6 +37,7 @@
 from vulnerabilities.importers.nginx import NginxImporter
 from vulnerabilities.importers.npm import NpmImporter
 from vulnerabilities.importers.oss_fuzz import OSSFuzzImporter
+from vulnerabilities.importers.ruby import RubyImporter
 from vulnerabilities.importers.ubuntu import UbuntuImporter
 from vulnerabilities.improver import MAX_CONFIDENCE
 from vulnerabilities.improver import Improver
@@ -460,3 +461,8 @@ class UbuntuOvalImprover(ValidVersionImprover):
 class OSSFuzzImprover(ValidVersionImprover):
     importer = OSSFuzzImporter
     ignorable_versions = []
+
+
+class RubyImprover(ValidVersionImprover):
+    importer = RubyImporter
+    ignorable_versions = []

vulnerabilities/tests/conftest.py

@@ -26,7 +26,6 @@ def no_rmtree(monkeypatch):
 # Step 3: Migrate all the tests
 collect_ignore = [
     "test_models.py",
-    "test_ruby.py",
     "test_rust.py",
     "test_suse_backports.py",
     "test_suse.py",

vulnerabilities/tests/test_data/ruby/CVE-2007-5770-expected.json

@@ -0,0 +1,42 @@
+{
+  "aliases": [
+    "CVE-2007-5770"
+  ],
+  "summary": "Ruby Net::HTTPS library does not validate server certificate CN\nThe (1) Net::ftptls, (2) Net::telnets, (3) Net::imap, (4) Net::pop, and (5)\nNet::smtp libraries in Ruby 1.8.5 and 1.8.6 do not verify that the\ncommonName (CN) field in a server certificate matches the domain name in a\nrequest sent over SSL, which makes it easier for remote attackers to\nintercept SSL transmissions via a man-in-the-middle attack or spoofed web\nsite, different components than CVE-2007-5162.",
+  "affected_packages": [
+    {
+      "package": {
+        "type": "ruby",
+        "namespace": null,
+        "name": "ruby",
+        "version": null,
+        "qualifiers": null,
+        "subpath": null
+      },
+      "affected_version_range": "vers:gem/<1.8.6.230|>=1.8.7",
+      "fixed_version": null
+    },
+    {
+      "package": {
+        "type": "ruby",
+        "namespace": null,
+        "name": "ruby",
+        "version": null,
+        "qualifiers": null,
+        "subpath": null
+      },
+      "affected_version_range": "vers:gem/<1.8.7",
+      "fixed_version": null
+    }
+  ],
+  "references": [
+    {
+      "reference_id": "",
+      "url": "http://www.cvedetails.com/cve/CVE-2007-5770/",
+      "severities": []
+    }
+  ],
+  "date_published": "2007-10-08T00:00:00+00:00",
+  "weaknesses": [],
+  "url": "https://github.com/rubysec/ruby-advisory-db"
+}

vulnerabilities/tests/test_data/ruby/CVE-2007-5770.yml

@@ -0,0 +1,17 @@
+---
+engine: ruby
+cve: 2007-5770
+url: http://www.cvedetails.com/cve/CVE-2007-5770/
+title: Ruby Net::HTTPS library does not validate server certificate CN
+date: 2007-10-08
+description: |
+  The (1) Net::ftptls, (2) Net::telnets, (3) Net::imap, (4) Net::pop, and (5)
+  Net::smtp libraries in Ruby 1.8.5 and 1.8.6 do not verify that the
+  commonName (CN) field in a server certificate matches the domain name in a
+  request sent over SSL, which makes it easier for remote attackers to
+  intercept SSL transmissions via a man-in-the-middle attack or spoofed web
+  site, different components than CVE-2007-5162.
+cvss_v2: 4.3
+patched_versions:
+- ~> 1.8.6.230
+- '>= 1.8.7'