Merge keshav-space:gitlab_datasource branch

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

vulntotal/datasources/gitlab.py

@@ -0,0 +1,182 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnTotal software is licensed under the Apache License version 2.0.
+# Data generated with VulnTotal require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnTotal or any VulnTotal
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnTotal and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnTotal should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnTotal is a free software tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+
+import json
+import logging
+import os
+import shutil
+import tarfile
+from pathlib import Path
+from typing import Iterable
+
+import requests
+import saneyaml
+from fetchcode import fetch
+from packageurl import PackageURL
+
+from vulntotal.validator import DataSource
+from vulntotal.validator import VendorData
+from vulntotal.vulntotal_utils import gitlab_constraints_satisfied
+
+logger = logging.getLogger(__name__)
+
+
+class GitlabDataSource(DataSource):
+    spdx_license_expression = "TODO"
+    license_url = "TODO"
+
+    def datasource_advisory(self, purl) -> Iterable[VendorData]:
+        package_slug = get_package_slug(purl)
+        location = download_subtree(package_slug, speculative_execution=True)
+        if not location:
+            clear_download(location)
+            path = self.supported_ecosystem()[purl.type]
+            casesensitive_package_slug = get_casesensitive_slug(path, package_slug)
+            location = download_subtree(casesensitive_package_slug)
+        if location:
+            interesting_advisories = parse_interesting_advisories(location, purl.version, delete_download=True)
+            return interesting_advisories
+        clear_download(location)
+
+    @classmethod
+    def supported_ecosystem(cls):
+        return {
+            "composer": "packagist",
+            "conan": "conan",
+            "gem": "gem",
+            "golang": "go",
+            "maven": "maven",
+            "npm": "npm",
+            "nuget": "nuget",
+            "pypi": "pypi",
+        }
+
+
+def get_package_slug(purl):
+    supported_ecosystem = GitlabDataSource.supported_ecosystem()
+
+    if purl.type not in supported_ecosystem:
+        return
+
+    ecosystem = supported_ecosystem[purl.type]
+    package_name = purl.name
+
+    if purl.type in ("maven", "composer", "golang"):
+        package_name = f"{purl.namespace}/{purl.name}"
+
+    return f"{ecosystem}/{package_name}"
+
+
+def download_subtree(package_slug: str, speculative_execution=False):
+    url = f"https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/archive/master/gemnasium-db-master.tar.gz?path={package_slug}"
+    response = fetch(url)
+    if os.path.getsize(response.location) > 0:
+        extracted_location = Path(response.location).parent.joinpath(
+            "temp_vulntotal_gitlab_datasource"
+        )
+        with tarfile.open(response.location, "r") as file_obj:
+            file_obj.extractall(extracted_location)
+        os.remove(response.location)
+        return extracted_location
+    if not speculative_execution:
+        logger.error(f"{package_slug} doesn't exist")
+    os.remove(response.location)
+
+
+def clear_download(location):
+    if location:
+        shutil.rmtree(location)
+
+
+def get_casesensitive_slug(path, package_slug):
+    payload = [
+        {
+            "operationName": "getPaginatedTree",
+            "variables": {
+                "projectPath": "gitlab-org/security-products/gemnasium-db",
+                "ref": "master",
+                "path": path,
+                "nextPageCursor": "",
+                "pageSize": 100,
+            },
+            "query": """
+            fragment TreeEntry on Entry { 
+                flatPath 
+            } 
+            query getPaginatedTree($projectPath: ID!, $path: String, $ref: String!, $nextPageCursor: String) { 
+                project(fullPath: $projectPath) { 
+                    repository { 
+                        paginatedTree(path: $path, ref: $ref, after: $nextPageCursor) { 
+                            pageInfo { 
+                            endCursor
+                            startCursor
+                            hasNextPage 
+                            } 
+                        nodes { 
+                            trees { 
+                                nodes { 
+                                    ...TreeEntry 
+                                    } 
+                                } 
+                            } 
+                        } 
+                    } 
+                } 
+            } """,
+        }
+    ]
+    url = "https://gitlab.com/api/graphql"
+    hasnext = True
+
+    while hasnext:
+        response = requests.post(url, json=payload).json()
+        paginated_tree = response[0]["data"]["project"]["repository"]["paginatedTree"]
+
+        for slug in paginated_tree["nodes"][0]["trees"]["nodes"]:
+            if slug["flatPath"].lower() == package_slug.lower():
+                return slug["flatPath"]
+
+            # If the namespace/subfolder contains multiple packages, then progressive transverse through folders tree
+            if package_slug.lower().startswith(slug["flatPath"].lower()):
+                return get_gitlab_style_slug(slug["flatPath"], package_slug)
+
+        payload[0]["variables"]["nextPageCursor"] = paginated_tree["pageInfo"]["endCursor"]
+        hasnext = paginated_tree["pageInfo"]["hasNextPage"]
+
+
+def parse_interesting_advisories(location, version, delete_download=False) -> Iterable[VendorData]:
+    path = Path(location)
+    glob = "**/*.yml"
+    files = (p for p in path.glob(glob) if p.is_file())
+    for file in sorted(files):
+        with open(file) as f:
+            gitlab_advisory = saneyaml.load(f)
+        if gitlab_constraints_satisfied(gitlab_advisory["affected_range"], version):
+            yield VendorData(
+                aliases=gitlab_advisory["identifiers"],
+                affected_versions=[gitlab_advisory["affected_range"]],
+                fixed_versions=gitlab_advisory["fixed_versions"],
+            )
+    if delete_download:
+        clear_download(location)

vulntotal/tests/test_data/gitlab/package_advisory_url-expected.json

@@ -0,0 +1,8 @@
+[
+  "pypi/jinja2",
+  "maven/org.apache.tomcat/tomcat",
+  "npm/semver-regex",
+  "go/github.com/mattermost/mattermost-server/v6/api4",
+  "packagist/bolt/core",
+  "nuget/moment.js"
+]

vulntotal/tests/test_data/gitlab/parsed_advisory-expected.json

@@ -0,0 +1,47 @@
+[
+  {
+    "affected_versions": [
+      "<=2.7.1"
+    ],
+    "fixed_versions": [
+      "2.7.2"
+    ],
+    "aliases": [
+      "CVE-2014-1402"
+    ]
+  },
+  {
+    "affected_versions": [
+      "<2.8.1"
+    ],
+    "fixed_versions": [
+      "2.8.1"
+    ],
+    "aliases": [
+      "GHSA-hj2j-77xm-mc5v",
+      "CVE-2016-10745"
+    ]
+  },
+  {
+    "affected_versions": [
+      "<2.10.1"
+    ],
+    "fixed_versions": [
+      "2.10.1"
+    ],
+    "aliases": [
+      "CVE-2019-10906"
+    ]
+  },
+  {
+    "affected_versions": [
+      "<2.11.3"
+    ],
+    "fixed_versions": [
+      "2.11.3"
+    ],
+    "aliases": [
+      "CVE-2020-28493"
+    ]
+  }
+]

vulntotal/tests/test_data/gitlab/purls.txt

@@ -0,0 +1,6 @@
+pkg:pypi/jinja2@2.4.1
+pkg:maven/org.apache.tomcat/tomcat@10.1.0
+pkg:npm/semver-regex@3.1.3
+pkg:golang/github.com/mattermost/mattermost-server/v6/api4@0.1
+pkg:composer/bolt/core@0.1
+pkg:nuget/moment.js@2.18.0

vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2014-1402.yml

@@ -0,0 +1,28 @@
+---
+identifier: "CVE-2014-1402"
+package_slug: "pypi/Jinja2"
+title: "Incorrect Default Permissions"
+description: "The default configuration for `bccache.FileSystemBytecodeCache` in Jinja2
+  before does not properly create temporary files, which allows local users to gain
+  privileges via a crafted `.cache` file with a name starting with `__jinja2_` in
+  `/tmp`."
+date: "2017-12-21"
+pubdate: "2014-05-19"
+affected_range: "<=2.7.1"
+fixed_versions:
+- "2.7.2"
+affected_versions: "All versions up to 2.7.1"
+not_impacted: "All versions after 2.7.1"
+solution: "Upgrade to version 2.7.2 or above."
+urls:
+- "https://bugzilla.redhat.com/CVE-2014-1402"
+- "http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=734747"
+- "https://github.com/mitsuhiko/jinja2/commit/acb672b6"
+cvss_v2: "AV:L/AC:M/Au:N/C:P/I:P/A:P"
+uuid: "73933300-77cd-49dd-8e3d-671763767b99"
+cwe_ids:
+- "CWE-1035"
+- "CWE-264"
+- "CWE-937"
+identifiers:
+- "CVE-2014-1402"

vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2016-10745.yml

@@ -0,0 +1,37 @@
+---
+identifier: "CVE-2016-10745"
+identifiers:
+- "GHSA-hj2j-77xm-mc5v"
+- "CVE-2016-10745"
+package_slug: "pypi/Jinja2"
+title: "Use of Externally-Controlled Format String"
+description: "In Pallets Jinja before 2.8.1, str.format allows a sandbox escape."
+date: "2021-09-14"
+pubdate: "2019-04-10"
+affected_range: "<2.8.1"
+fixed_versions:
+- "2.8.1"
+affected_versions: "All versions before 2.8.1"
+not_impacted: "All versions starting from 2.8.1"
+solution: "Upgrade to version 2.8.1 or above."
+urls:
+- "https://nvd.nist.gov/vuln/detail/CVE-2016-10745"
+- "https://github.com/pallets/jinja/commit/9b53045c34e61013dc8f09b7e52a555fa16bed16"
+- "https://access.redhat.com/errata/RHSA-2019:1022"
+- "https://access.redhat.com/errata/RHSA-2019:1237"
+- "https://access.redhat.com/errata/RHSA-2019:1260"
+- "https://access.redhat.com/errata/RHSA-2019:3964"
+- "https://access.redhat.com/errata/RHSA-2019:4062"
+- "https://github.com/advisories/GHSA-hj2j-77xm-mc5v"
+- "https://palletsprojects.com/blog/jinja-281-released/"
+- "https://usn.ubuntu.com/4011-1/"
+- "https://usn.ubuntu.com/4011-2/"
+- "http://lists.opensuse.org/opensuse-security-announce/2019-05/msg00030.html"
+- "http://lists.opensuse.org/opensuse-security-announce/2019-06/msg00064.html"
+cvss_v2: "AV:N/AC:L/Au:N/C:P/I:N/A:N"
+cvss_v3: "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:N/A:N"
+uuid: "db196e91-7fc4-4b94-93b3-e0c0dc01bcbe"
+cwe_ids:
+- "CWE-1035"
+- "CWE-134"
+- "CWE-937"

vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2019-10906.yml

@@ -0,0 +1,24 @@
+---
+identifier: "CVE-2019-10906"
+package_slug: "pypi/Jinja2"
+title: "Sandbox Escape"
+description: "In Pallets Jinja, str.format_map allows a sandbox escape."
+date: "2019-06-06"
+pubdate: "2019-04-06"
+affected_range: "<2.10.1"
+fixed_versions:
+- "2.10.1"
+affected_versions: "All versions before 2.10.1"
+not_impacted: "All versions starting from 2.10.1"
+solution: "Upgrade to version 2.10.1 or above."
+urls:
+- "https://nvd.nist.gov/vuln/detail/CVE-2019-10906"
+- "https://palletsprojects.com/blog/jinja-2-10-1-released"
+cvss_v2: "AV:N/AC:L/Au:N/C:P/I:N/A:N"
+cvss_v3: "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:N/A:N"
+uuid: "b2d59abd-b748-4d93-bb21-55f9e9aecea2"
+cwe_ids:
+- "CWE-1035"
+- "CWE-937"
+identifiers:
+- "CVE-2019-10906"

vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2019-8341.yml

@@ -0,0 +1,28 @@
+---
+identifier: "CVE-2019-8341"
+package_slug: "pypi/Jinja2"
+title: "Code Injection"
+description: "The `from_string` function is prone to Server Side Template Injection
+  (SSTI) where it takes the `source` parameter as a template object, renders it, and
+  then returns it. The attacker can exploit it with `{{INJECTION COMMANDS}}` in a
+  URI."
+date: "2019-08-06"
+pubdate: "2019-02-15"
+affected_range: "==2.10"
+fixed_versions:
+- "2.10.1"
+affected_versions: "Version 2.10"
+not_impacted: "All versions before 2.10, all versions after 2.10"
+solution: "Upgrade to version 2.10.1 or above."
+urls:
+- "https://nvd.nist.gov/vuln/detail/CVE-2019-8341"
+- "https://www.exploit-db.com/exploits/46386/"
+cvss_v2: "AV:N/AC:L/Au:N/C:P/I:P/A:P"
+cvss_v3: "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"
+uuid: "6b00a76b-aac8-4ce1-b7b2-122a73f22985"
+cwe_ids:
+- "CWE-1035"
+- "CWE-937"
+- "CWE-94"
+identifiers:
+- "CVE-2019-8341"

vulntotal/tests/test_data/gitlab/temp_vulntotal_gitlab_datasource/gemnasium-db-master-pypi-Jinja2/pypi/Jinja2/CVE-2020-28493.yml

@@ -0,0 +1,26 @@
+---
+identifier: "CVE-2020-28493"
+package_slug: "pypi/Jinja2"
+title: "Regular Expression Denial of Service"
+description: "The ReDOS vulnerability of the regex is mainly due to the sub-pattern
+  `[a-zA-Z0-9._-]+.[a-zA-Z0-9._-]+` This issue can be mitigated by Markdown to format
+  user content instead of the urlize filter, or by implementing request timeouts and
+  limiting process memory."
+date: "2021-07-21"
+pubdate: "2021-02-01"
+affected_range: "<2.11.3"
+fixed_versions:
+- "2.11.3"
+affected_versions: "All versions before 2.11.3"
+not_impacted: "All versions starting from 2.11.3"
+solution: "Upgrade to version 2.11.3 or above."
+urls:
+- "https://nvd.nist.gov/vuln/detail/CVE-2020-28493"
+cvss_v2: "AV:N/AC:L/Au:N/C:N/I:N/A:P"
+cvss_v3: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:L"
+uuid: "17c4602a-c0ae-48d0-96f6-08c84d55d0cb"
+cwe_ids:
+- "CWE-1035"
+- "CWE-937"
+identifiers:
+- "CVE-2020-28493"