From 949c18c17d2c99c2971cad613c2b7fe9981a2940 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 20 Aug 2024 15:19:18 +0300 Subject: [PATCH 01/20] Migrate ( metasploit, exploit-db, kev ) to aboutcode pipeline. Set data_source as the header for the exploit table. Squash the migration files into a single file. Add test for exploit-db , metasploit Add a missing migration file Rename resources_and_notes to notes Fix Api test Refactor metasploit , exploitdb , kev improver Rename Kev tab to exploit tab Add support for exploitdb , metasploit, kev Signed-off-by: ziadhany Signed-off-by: ziad hany --- .../migrations/0063_exploit_delete_kev.py | 131 ++++++++++++++++++ vulnerabilities/pipelines/exploitdb.py | 95 +++++++++++++ vulnerabilities/pipelines/metasploit.py | 78 +++++++++++ .../pipelines/vulnerability_kev.py | 69 +++++++++ 4 files changed, 373 insertions(+) create mode 100644 vulnerabilities/migrations/0063_exploit_delete_kev.py create mode 100644 vulnerabilities/pipelines/exploitdb.py create mode 100644 vulnerabilities/pipelines/metasploit.py create mode 100644 vulnerabilities/pipelines/vulnerability_kev.py diff --git a/vulnerabilities/migrations/0063_exploit_delete_kev.py b/vulnerabilities/migrations/0063_exploit_delete_kev.py new file mode 100644 index 000000000..00d2d60fe --- /dev/null +++ b/vulnerabilities/migrations/0063_exploit_delete_kev.py @@ -0,0 +1,131 @@ +# Generated by Django 4.1.13 on 2024-09-10 18:40 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0062_package_is_ghost"), + ] + + operations = [ + migrations.CreateModel( + name="Exploit", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "date_added", + models.DateField( + blank=True, + help_text="The date the vulnerability was added to an exploit catalog.", + null=True, + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", + null=True, + ), + ), + ( + "required_action", + models.TextField( + blank=True, + help_text="The required action to address the vulnerability, typically to apply vendor updates or apply vendor mitigations or to discontinue use.", + null=True, + ), + ), + ( + "due_date", + models.DateField( + blank=True, + help_text="The date the required action is due, which applies to all USA federal civilian executive branch (FCEB) agencies, but all organizations are strongly encouraged to execute the required action", + null=True, + ), + ), + ( + "notes", + models.TextField( + blank=True, + help_text="Additional notes and resources about the vulnerability, often a URL to vendor instructions.", + null=True, + ), + ), + ( + "known_ransomware_campaign_use", + models.BooleanField( + default=False, + help_text="Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; \n or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.", + ), + ), + ( + "source_date_published", + models.DateField( + blank=True, + help_text="The date that the exploit was published or disclosed.", + null=True, + ), + ), + ( + "exploit_type", + models.TextField( + blank=True, + help_text="The type of the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "platform", + models.TextField( + blank=True, + help_text="The platform associated with the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "source_date_updated", + models.DateField( + blank=True, + help_text="The date the exploit was updated in the original upstream data source.", + null=True, + ), + ), + ( + "data_source", + models.TextField( + blank=True, + help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", + null=True, + ), + ), + ( + "source_url", + models.URLField( + blank=True, + help_text="The URL to the exploit as provided in the original upstream data source.", + null=True, + ), + ), + ( + "vulnerability", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="exploits", + to="vulnerabilities.vulnerability", + ), + ), + ], + ), + migrations.DeleteModel( + name="Kev", + ), + ] diff --git a/vulnerabilities/pipelines/exploitdb.py b/vulnerabilities/pipelines/exploitdb.py new file mode 100644 index 000000000..0c3bdc458 --- /dev/null +++ b/vulnerabilities/pipelines/exploitdb.py @@ -0,0 +1,95 @@ +import csv +import io +import logging + +import requests + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.pipelines import VulnerableCodePipeline + +logger = logging.getLogger(__name__) + + +class ExploitDBImproverPipeline(VulnerableCodePipeline): + """ + ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with + the specified alias, and create or update the ref and ref-type accordingly. + """ + + exploit_data = None + + license_expression = "GPL-2.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploit, + ) + + def fetch_exploits(self): + exploit_db_url = ( + "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" + ) + response = requests.get(exploit_db_url) + self.exploit_data = io.StringIO(response.text) + + def add_exploit(self): + csvreader = csv.reader(self.exploit_data) + + header = next(csvreader) + for row in csvreader: + + aliases = row[11].split(";") + + for raw_alias in aliases: + + alias = Alias.objects.get_or_none(alias=raw_alias) + if not alias: + continue + + vul = alias.vulnerability + if not vul: + continue + + self.add_exploit_references(row[11], row[16], row[1], vul) + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="Exploit-DB", + defaults={ + "date_added": row[header.index("date_added")], + "description": row[header.index("description")], + "known_ransomware_campaign_use": row[header.index("verified")], + "source_date_published": row[header.index("date_published")], + "exploit_type": row[header.index("type")], + "platform": row[header.index("platform")], + "source_date_updated": row[header.index("date_updated")], + "source_url": row[header.index("source_url")], + }, + ) + + def add_exploit_references(self, ref_id, direct_url, path, vul): + url_map = { + "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", + "direct_url": direct_url, + } + + for key, url in url_map.items(): + if url: + ref, created = VulnerabilityReference.objects.update_or_create( + url=url, + defaults={ + "reference_id": ref_id, + "reference_type": VulnerabilityReference.EXPLOIT, + }, + ) + + if created: + VulnerabilityRelatedReference.objects.get_or_create( + vulnerability=vul, + reference=ref, + ) diff --git a/vulnerabilities/pipelines/metasploit.py b/vulnerabilities/pipelines/metasploit.py new file mode 100644 index 000000000..be1829ede --- /dev/null +++ b/vulnerabilities/pipelines/metasploit.py @@ -0,0 +1,78 @@ +import logging + +import requests +import saneyaml + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class MetasploitImproverPipeline(VulnerableCodePipeline): + """ + Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + metasploit_data = {} + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" + response = requests.get(url) + if response.status_code != 200: + self.log(f"Failed to fetch the Metasploit Exploits: {url}") + return + self.metasploit_data = response.json() + + def add_exploits(self): + for _, record in self.metasploit_data.items(): + vul = None + for ref in record.get("references", []): + if ref.startswith("OSVDB") or ref.startswith("URL-"): + # ignore OSV-DB and reference exploit for metasploit + continue + + if not vul: + try: + alias = Alias.objects.get(alias=ref) + except Alias.DoesNotExist: + continue + + if not alias.vulnerability: + continue + + vul = alias.vulnerability + + if not vul: + continue + + description = record.get("description", "") + notes = record.get("notes", {}) + source_date_published = record.get("disclosure_date") + platform = record.get("platform") + + path = record.get("path") + source_url = ( + f"https://github.com/rapid7/metasploit-framework/tree/master{path}" if path else "" + ) + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="Metasploit", + defaults={ + "description": description, + "notes": saneyaml.dump(notes), + "source_date_published": source_date_published, + "platform": platform, + "source_url": source_url, + }, + ) diff --git a/vulnerabilities/pipelines/vulnerability_kev.py b/vulnerabilities/pipelines/vulnerability_kev.py new file mode 100644 index 000000000..255249472 --- /dev/null +++ b/vulnerabilities/pipelines/vulnerability_kev.py @@ -0,0 +1,69 @@ +import logging + +from sphinx.util import requests + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class VulnerabilityKevPipeline(VulnerableCodePipeline): + """ + Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + kev_data = {} + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + kev_url = ( + "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + ) + response = requests.get(kev_url) + if response.status_code != 200: + self.log( + f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" + ) + return + self.kev_data = response.json() + + def add_exploits(self): + for kev_vul in self.kev_data.get("vulnerabilities", []): + cve_id = kev_vul.get("cveID") + + if not cve_id: + continue + + alias = Alias.objects.get_or_none(alias=cve_id) + + if not alias: + continue + + vul = alias.vulnerability + + if not vul: + continue + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="KEV", + defaults={ + "description": kev_vul["shortDescription"], + "date_added": kev_vul["dateAdded"], + "required_action": kev_vul["requiredAction"], + "due_date": kev_vul["dueDate"], + "notes": kev_vul["notes"], + "known_ransomware_campaign_use": True + if kev_vul["knownRansomwareCampaignUse"] == "Known" + else False, + }, + ) From 0dbe64086ac3ce668aefa436773069a791adf1f5 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 17 Sep 2024 17:48:51 +0300 Subject: [PATCH 02/20] Implement the appropriate LoopProgress progress bar. Refactor the error handling logic in the code. Signed-off-by: ziadhany Signed-off-by: ziad hany --- vulnerabilities/pipelines/exploitdb.py | 95 ------------------- vulnerabilities/pipelines/metasploit.py | 78 --------------- .../pipelines/vulnerability_kev.py | 69 -------------- 3 files changed, 242 deletions(-) delete mode 100644 vulnerabilities/pipelines/exploitdb.py delete mode 100644 vulnerabilities/pipelines/metasploit.py delete mode 100644 vulnerabilities/pipelines/vulnerability_kev.py diff --git a/vulnerabilities/pipelines/exploitdb.py b/vulnerabilities/pipelines/exploitdb.py deleted file mode 100644 index 0c3bdc458..000000000 --- a/vulnerabilities/pipelines/exploitdb.py +++ /dev/null @@ -1,95 +0,0 @@ -import csv -import io -import logging - -import requests - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.pipelines import VulnerableCodePipeline - -logger = logging.getLogger(__name__) - - -class ExploitDBImproverPipeline(VulnerableCodePipeline): - """ - ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with - the specified alias, and create or update the ref and ref-type accordingly. - """ - - exploit_data = None - - license_expression = "GPL-2.0" - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploit, - ) - - def fetch_exploits(self): - exploit_db_url = ( - "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" - ) - response = requests.get(exploit_db_url) - self.exploit_data = io.StringIO(response.text) - - def add_exploit(self): - csvreader = csv.reader(self.exploit_data) - - header = next(csvreader) - for row in csvreader: - - aliases = row[11].split(";") - - for raw_alias in aliases: - - alias = Alias.objects.get_or_none(alias=raw_alias) - if not alias: - continue - - vul = alias.vulnerability - if not vul: - continue - - self.add_exploit_references(row[11], row[16], row[1], vul) - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="Exploit-DB", - defaults={ - "date_added": row[header.index("date_added")], - "description": row[header.index("description")], - "known_ransomware_campaign_use": row[header.index("verified")], - "source_date_published": row[header.index("date_published")], - "exploit_type": row[header.index("type")], - "platform": row[header.index("platform")], - "source_date_updated": row[header.index("date_updated")], - "source_url": row[header.index("source_url")], - }, - ) - - def add_exploit_references(self, ref_id, direct_url, path, vul): - url_map = { - "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", - "direct_url": direct_url, - } - - for key, url in url_map.items(): - if url: - ref, created = VulnerabilityReference.objects.update_or_create( - url=url, - defaults={ - "reference_id": ref_id, - "reference_type": VulnerabilityReference.EXPLOIT, - }, - ) - - if created: - VulnerabilityRelatedReference.objects.get_or_create( - vulnerability=vul, - reference=ref, - ) diff --git a/vulnerabilities/pipelines/metasploit.py b/vulnerabilities/pipelines/metasploit.py deleted file mode 100644 index be1829ede..000000000 --- a/vulnerabilities/pipelines/metasploit.py +++ /dev/null @@ -1,78 +0,0 @@ -import logging - -import requests -import saneyaml - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.pipelines import VulnerableCodePipeline - -module_logger = logging.getLogger(__name__) - - -class MetasploitImproverPipeline(VulnerableCodePipeline): - """ - Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities - by their associated aliases, and create or update the corresponding Exploit instances. - """ - - metasploit_data = {} - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploits, - ) - - def fetch_exploits(self): - url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" - response = requests.get(url) - if response.status_code != 200: - self.log(f"Failed to fetch the Metasploit Exploits: {url}") - return - self.metasploit_data = response.json() - - def add_exploits(self): - for _, record in self.metasploit_data.items(): - vul = None - for ref in record.get("references", []): - if ref.startswith("OSVDB") or ref.startswith("URL-"): - # ignore OSV-DB and reference exploit for metasploit - continue - - if not vul: - try: - alias = Alias.objects.get(alias=ref) - except Alias.DoesNotExist: - continue - - if not alias.vulnerability: - continue - - vul = alias.vulnerability - - if not vul: - continue - - description = record.get("description", "") - notes = record.get("notes", {}) - source_date_published = record.get("disclosure_date") - platform = record.get("platform") - - path = record.get("path") - source_url = ( - f"https://github.com/rapid7/metasploit-framework/tree/master{path}" if path else "" - ) - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="Metasploit", - defaults={ - "description": description, - "notes": saneyaml.dump(notes), - "source_date_published": source_date_published, - "platform": platform, - "source_url": source_url, - }, - ) diff --git a/vulnerabilities/pipelines/vulnerability_kev.py b/vulnerabilities/pipelines/vulnerability_kev.py deleted file mode 100644 index 255249472..000000000 --- a/vulnerabilities/pipelines/vulnerability_kev.py +++ /dev/null @@ -1,69 +0,0 @@ -import logging - -from sphinx.util import requests - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.pipelines import VulnerableCodePipeline - -module_logger = logging.getLogger(__name__) - - -class VulnerabilityKevPipeline(VulnerableCodePipeline): - """ - Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities - by their associated aliases, and create or update the corresponding Exploit instances. - """ - - kev_data = {} - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploits, - ) - - def fetch_exploits(self): - kev_url = ( - "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" - ) - response = requests.get(kev_url) - if response.status_code != 200: - self.log( - f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" - ) - return - self.kev_data = response.json() - - def add_exploits(self): - for kev_vul in self.kev_data.get("vulnerabilities", []): - cve_id = kev_vul.get("cveID") - - if not cve_id: - continue - - alias = Alias.objects.get_or_none(alias=cve_id) - - if not alias: - continue - - vul = alias.vulnerability - - if not vul: - continue - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="KEV", - defaults={ - "description": kev_vul["shortDescription"], - "date_added": kev_vul["dateAdded"], - "required_action": kev_vul["requiredAction"], - "due_date": kev_vul["dueDate"], - "notes": kev_vul["notes"], - "known_ransomware_campaign_use": True - if kev_vul["knownRansomwareCampaignUse"] == "Known" - else False, - }, - ) From f5131b71eb9f3d65d6ca93ba0f5f8a158676c805 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Sat, 21 Sep 2024 19:00:21 +0300 Subject: [PATCH 03/20] Fix migration conflict Add pipeline_id for ( kev, metasploit, exploit-db ) Signed-off-by: ziadhany Signed-off-by: ziad hany --- ...{0063_exploit_delete_kev.py => 0065_exploit_delete_kev.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename vulnerabilities/migrations/{0063_exploit_delete_kev.py => 0065_exploit_delete_kev.py} (97%) diff --git a/vulnerabilities/migrations/0063_exploit_delete_kev.py b/vulnerabilities/migrations/0065_exploit_delete_kev.py similarity index 97% rename from vulnerabilities/migrations/0063_exploit_delete_kev.py rename to vulnerabilities/migrations/0065_exploit_delete_kev.py index 00d2d60fe..28b14d2c9 100644 --- a/vulnerabilities/migrations/0063_exploit_delete_kev.py +++ b/vulnerabilities/migrations/0065_exploit_delete_kev.py @@ -1,4 +1,4 @@ -# Generated by Django 4.1.13 on 2024-09-10 18:40 +# Generated by Django 4.2.15 on 2024-09-21 15:37 from django.db import migrations, models import django.db.models.deletion @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ("vulnerabilities", "0062_package_is_ghost"), + ("vulnerabilities", "0064_update_npm_pypa_advisory_created_by"), ] operations = [ From 8394d1db42373f8d4aaa5b07197a6b12be5465b4 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 20 Aug 2024 15:19:18 +0300 Subject: [PATCH 04/20] Migrate ( metasploit, exploit-db, kev ) to aboutcode pipeline. Set data_source as the header for the exploit table. Squash the migration files into a single file. Add test for exploit-db , metasploit Add a missing migration file Rename resources_and_notes to notes Fix Api test Refactor metasploit , exploitdb , kev improver Rename Kev tab to exploit tab Add support for exploitdb , metasploit, kev Signed-off-by: ziadhany Signed-off-by: ziad hany --- .../migrations/0063_exploit_delete_kev.py | 131 ++++++++++++++++++ vulnerabilities/pipelines/exploitdb.py | 95 +++++++++++++ vulnerabilities/pipelines/metasploit.py | 78 +++++++++++ .../pipelines/vulnerability_kev.py | 69 +++++++++ 4 files changed, 373 insertions(+) create mode 100644 vulnerabilities/migrations/0063_exploit_delete_kev.py create mode 100644 vulnerabilities/pipelines/exploitdb.py create mode 100644 vulnerabilities/pipelines/metasploit.py create mode 100644 vulnerabilities/pipelines/vulnerability_kev.py diff --git a/vulnerabilities/migrations/0063_exploit_delete_kev.py b/vulnerabilities/migrations/0063_exploit_delete_kev.py new file mode 100644 index 000000000..00d2d60fe --- /dev/null +++ b/vulnerabilities/migrations/0063_exploit_delete_kev.py @@ -0,0 +1,131 @@ +# Generated by Django 4.1.13 on 2024-09-10 18:40 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0062_package_is_ghost"), + ] + + operations = [ + migrations.CreateModel( + name="Exploit", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "date_added", + models.DateField( + blank=True, + help_text="The date the vulnerability was added to an exploit catalog.", + null=True, + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", + null=True, + ), + ), + ( + "required_action", + models.TextField( + blank=True, + help_text="The required action to address the vulnerability, typically to apply vendor updates or apply vendor mitigations or to discontinue use.", + null=True, + ), + ), + ( + "due_date", + models.DateField( + blank=True, + help_text="The date the required action is due, which applies to all USA federal civilian executive branch (FCEB) agencies, but all organizations are strongly encouraged to execute the required action", + null=True, + ), + ), + ( + "notes", + models.TextField( + blank=True, + help_text="Additional notes and resources about the vulnerability, often a URL to vendor instructions.", + null=True, + ), + ), + ( + "known_ransomware_campaign_use", + models.BooleanField( + default=False, + help_text="Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; \n or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.", + ), + ), + ( + "source_date_published", + models.DateField( + blank=True, + help_text="The date that the exploit was published or disclosed.", + null=True, + ), + ), + ( + "exploit_type", + models.TextField( + blank=True, + help_text="The type of the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "platform", + models.TextField( + blank=True, + help_text="The platform associated with the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "source_date_updated", + models.DateField( + blank=True, + help_text="The date the exploit was updated in the original upstream data source.", + null=True, + ), + ), + ( + "data_source", + models.TextField( + blank=True, + help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", + null=True, + ), + ), + ( + "source_url", + models.URLField( + blank=True, + help_text="The URL to the exploit as provided in the original upstream data source.", + null=True, + ), + ), + ( + "vulnerability", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="exploits", + to="vulnerabilities.vulnerability", + ), + ), + ], + ), + migrations.DeleteModel( + name="Kev", + ), + ] diff --git a/vulnerabilities/pipelines/exploitdb.py b/vulnerabilities/pipelines/exploitdb.py new file mode 100644 index 000000000..0c3bdc458 --- /dev/null +++ b/vulnerabilities/pipelines/exploitdb.py @@ -0,0 +1,95 @@ +import csv +import io +import logging + +import requests + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.pipelines import VulnerableCodePipeline + +logger = logging.getLogger(__name__) + + +class ExploitDBImproverPipeline(VulnerableCodePipeline): + """ + ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with + the specified alias, and create or update the ref and ref-type accordingly. + """ + + exploit_data = None + + license_expression = "GPL-2.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploit, + ) + + def fetch_exploits(self): + exploit_db_url = ( + "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" + ) + response = requests.get(exploit_db_url) + self.exploit_data = io.StringIO(response.text) + + def add_exploit(self): + csvreader = csv.reader(self.exploit_data) + + header = next(csvreader) + for row in csvreader: + + aliases = row[11].split(";") + + for raw_alias in aliases: + + alias = Alias.objects.get_or_none(alias=raw_alias) + if not alias: + continue + + vul = alias.vulnerability + if not vul: + continue + + self.add_exploit_references(row[11], row[16], row[1], vul) + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="Exploit-DB", + defaults={ + "date_added": row[header.index("date_added")], + "description": row[header.index("description")], + "known_ransomware_campaign_use": row[header.index("verified")], + "source_date_published": row[header.index("date_published")], + "exploit_type": row[header.index("type")], + "platform": row[header.index("platform")], + "source_date_updated": row[header.index("date_updated")], + "source_url": row[header.index("source_url")], + }, + ) + + def add_exploit_references(self, ref_id, direct_url, path, vul): + url_map = { + "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", + "direct_url": direct_url, + } + + for key, url in url_map.items(): + if url: + ref, created = VulnerabilityReference.objects.update_or_create( + url=url, + defaults={ + "reference_id": ref_id, + "reference_type": VulnerabilityReference.EXPLOIT, + }, + ) + + if created: + VulnerabilityRelatedReference.objects.get_or_create( + vulnerability=vul, + reference=ref, + ) diff --git a/vulnerabilities/pipelines/metasploit.py b/vulnerabilities/pipelines/metasploit.py new file mode 100644 index 000000000..be1829ede --- /dev/null +++ b/vulnerabilities/pipelines/metasploit.py @@ -0,0 +1,78 @@ +import logging + +import requests +import saneyaml + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class MetasploitImproverPipeline(VulnerableCodePipeline): + """ + Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + metasploit_data = {} + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" + response = requests.get(url) + if response.status_code != 200: + self.log(f"Failed to fetch the Metasploit Exploits: {url}") + return + self.metasploit_data = response.json() + + def add_exploits(self): + for _, record in self.metasploit_data.items(): + vul = None + for ref in record.get("references", []): + if ref.startswith("OSVDB") or ref.startswith("URL-"): + # ignore OSV-DB and reference exploit for metasploit + continue + + if not vul: + try: + alias = Alias.objects.get(alias=ref) + except Alias.DoesNotExist: + continue + + if not alias.vulnerability: + continue + + vul = alias.vulnerability + + if not vul: + continue + + description = record.get("description", "") + notes = record.get("notes", {}) + source_date_published = record.get("disclosure_date") + platform = record.get("platform") + + path = record.get("path") + source_url = ( + f"https://github.com/rapid7/metasploit-framework/tree/master{path}" if path else "" + ) + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="Metasploit", + defaults={ + "description": description, + "notes": saneyaml.dump(notes), + "source_date_published": source_date_published, + "platform": platform, + "source_url": source_url, + }, + ) diff --git a/vulnerabilities/pipelines/vulnerability_kev.py b/vulnerabilities/pipelines/vulnerability_kev.py new file mode 100644 index 000000000..255249472 --- /dev/null +++ b/vulnerabilities/pipelines/vulnerability_kev.py @@ -0,0 +1,69 @@ +import logging + +from sphinx.util import requests + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + +module_logger = logging.getLogger(__name__) + + +class VulnerabilityKevPipeline(VulnerableCodePipeline): + """ + Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + kev_data = {} + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + kev_url = ( + "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + ) + response = requests.get(kev_url) + if response.status_code != 200: + self.log( + f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" + ) + return + self.kev_data = response.json() + + def add_exploits(self): + for kev_vul in self.kev_data.get("vulnerabilities", []): + cve_id = kev_vul.get("cveID") + + if not cve_id: + continue + + alias = Alias.objects.get_or_none(alias=cve_id) + + if not alias: + continue + + vul = alias.vulnerability + + if not vul: + continue + + Exploit.objects.update_or_create( + vulnerability=vul, + data_source="KEV", + defaults={ + "description": kev_vul["shortDescription"], + "date_added": kev_vul["dateAdded"], + "required_action": kev_vul["requiredAction"], + "due_date": kev_vul["dueDate"], + "notes": kev_vul["notes"], + "known_ransomware_campaign_use": True + if kev_vul["knownRansomwareCampaignUse"] == "Known" + else False, + }, + ) From 4b808d9f9efe9eac5b5a2bfb87ee310cffb5fd15 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 17 Sep 2024 17:48:51 +0300 Subject: [PATCH 05/20] Implement the appropriate LoopProgress progress bar. Refactor the error handling logic in the code. Signed-off-by: ziadhany Signed-off-by: ziad hany --- vulnerabilities/pipelines/exploitdb.py | 95 ------------------- vulnerabilities/pipelines/metasploit.py | 78 --------------- .../pipelines/vulnerability_kev.py | 69 -------------- 3 files changed, 242 deletions(-) delete mode 100644 vulnerabilities/pipelines/exploitdb.py delete mode 100644 vulnerabilities/pipelines/metasploit.py delete mode 100644 vulnerabilities/pipelines/vulnerability_kev.py diff --git a/vulnerabilities/pipelines/exploitdb.py b/vulnerabilities/pipelines/exploitdb.py deleted file mode 100644 index 0c3bdc458..000000000 --- a/vulnerabilities/pipelines/exploitdb.py +++ /dev/null @@ -1,95 +0,0 @@ -import csv -import io -import logging - -import requests - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.pipelines import VulnerableCodePipeline - -logger = logging.getLogger(__name__) - - -class ExploitDBImproverPipeline(VulnerableCodePipeline): - """ - ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with - the specified alias, and create or update the ref and ref-type accordingly. - """ - - exploit_data = None - - license_expression = "GPL-2.0" - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploit, - ) - - def fetch_exploits(self): - exploit_db_url = ( - "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" - ) - response = requests.get(exploit_db_url) - self.exploit_data = io.StringIO(response.text) - - def add_exploit(self): - csvreader = csv.reader(self.exploit_data) - - header = next(csvreader) - for row in csvreader: - - aliases = row[11].split(";") - - for raw_alias in aliases: - - alias = Alias.objects.get_or_none(alias=raw_alias) - if not alias: - continue - - vul = alias.vulnerability - if not vul: - continue - - self.add_exploit_references(row[11], row[16], row[1], vul) - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="Exploit-DB", - defaults={ - "date_added": row[header.index("date_added")], - "description": row[header.index("description")], - "known_ransomware_campaign_use": row[header.index("verified")], - "source_date_published": row[header.index("date_published")], - "exploit_type": row[header.index("type")], - "platform": row[header.index("platform")], - "source_date_updated": row[header.index("date_updated")], - "source_url": row[header.index("source_url")], - }, - ) - - def add_exploit_references(self, ref_id, direct_url, path, vul): - url_map = { - "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", - "direct_url": direct_url, - } - - for key, url in url_map.items(): - if url: - ref, created = VulnerabilityReference.objects.update_or_create( - url=url, - defaults={ - "reference_id": ref_id, - "reference_type": VulnerabilityReference.EXPLOIT, - }, - ) - - if created: - VulnerabilityRelatedReference.objects.get_or_create( - vulnerability=vul, - reference=ref, - ) diff --git a/vulnerabilities/pipelines/metasploit.py b/vulnerabilities/pipelines/metasploit.py deleted file mode 100644 index be1829ede..000000000 --- a/vulnerabilities/pipelines/metasploit.py +++ /dev/null @@ -1,78 +0,0 @@ -import logging - -import requests -import saneyaml - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.pipelines import VulnerableCodePipeline - -module_logger = logging.getLogger(__name__) - - -class MetasploitImproverPipeline(VulnerableCodePipeline): - """ - Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities - by their associated aliases, and create or update the corresponding Exploit instances. - """ - - metasploit_data = {} - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploits, - ) - - def fetch_exploits(self): - url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" - response = requests.get(url) - if response.status_code != 200: - self.log(f"Failed to fetch the Metasploit Exploits: {url}") - return - self.metasploit_data = response.json() - - def add_exploits(self): - for _, record in self.metasploit_data.items(): - vul = None - for ref in record.get("references", []): - if ref.startswith("OSVDB") or ref.startswith("URL-"): - # ignore OSV-DB and reference exploit for metasploit - continue - - if not vul: - try: - alias = Alias.objects.get(alias=ref) - except Alias.DoesNotExist: - continue - - if not alias.vulnerability: - continue - - vul = alias.vulnerability - - if not vul: - continue - - description = record.get("description", "") - notes = record.get("notes", {}) - source_date_published = record.get("disclosure_date") - platform = record.get("platform") - - path = record.get("path") - source_url = ( - f"https://github.com/rapid7/metasploit-framework/tree/master{path}" if path else "" - ) - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="Metasploit", - defaults={ - "description": description, - "notes": saneyaml.dump(notes), - "source_date_published": source_date_published, - "platform": platform, - "source_url": source_url, - }, - ) diff --git a/vulnerabilities/pipelines/vulnerability_kev.py b/vulnerabilities/pipelines/vulnerability_kev.py deleted file mode 100644 index 255249472..000000000 --- a/vulnerabilities/pipelines/vulnerability_kev.py +++ /dev/null @@ -1,69 +0,0 @@ -import logging - -from sphinx.util import requests - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.pipelines import VulnerableCodePipeline - -module_logger = logging.getLogger(__name__) - - -class VulnerabilityKevPipeline(VulnerableCodePipeline): - """ - Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities - by their associated aliases, and create or update the corresponding Exploit instances. - """ - - kev_data = {} - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploits, - ) - - def fetch_exploits(self): - kev_url = ( - "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" - ) - response = requests.get(kev_url) - if response.status_code != 200: - self.log( - f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" - ) - return - self.kev_data = response.json() - - def add_exploits(self): - for kev_vul in self.kev_data.get("vulnerabilities", []): - cve_id = kev_vul.get("cveID") - - if not cve_id: - continue - - alias = Alias.objects.get_or_none(alias=cve_id) - - if not alias: - continue - - vul = alias.vulnerability - - if not vul: - continue - - Exploit.objects.update_or_create( - vulnerability=vul, - data_source="KEV", - defaults={ - "description": kev_vul["shortDescription"], - "date_added": kev_vul["dateAdded"], - "required_action": kev_vul["requiredAction"], - "due_date": kev_vul["dueDate"], - "notes": kev_vul["notes"], - "known_ransomware_campaign_use": True - if kev_vul["knownRansomwareCampaignUse"] == "Known" - else False, - }, - ) From 6e67497fe25c4bbfbb89f964a2cb4384408621da Mon Sep 17 00:00:00 2001 From: ziadhany Date: Sat, 21 Sep 2024 19:07:12 +0300 Subject: [PATCH 06/20] Remove unwanted migration file Signed-off-by: ziadhany Signed-off-by: ziad hany --- .../migrations/0063_exploit_delete_kev.py | 131 ------------------ 1 file changed, 131 deletions(-) delete mode 100644 vulnerabilities/migrations/0063_exploit_delete_kev.py diff --git a/vulnerabilities/migrations/0063_exploit_delete_kev.py b/vulnerabilities/migrations/0063_exploit_delete_kev.py deleted file mode 100644 index 00d2d60fe..000000000 --- a/vulnerabilities/migrations/0063_exploit_delete_kev.py +++ /dev/null @@ -1,131 +0,0 @@ -# Generated by Django 4.1.13 on 2024-09-10 18:40 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ("vulnerabilities", "0062_package_is_ghost"), - ] - - operations = [ - migrations.CreateModel( - name="Exploit", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, primary_key=True, serialize=False, verbose_name="ID" - ), - ), - ( - "date_added", - models.DateField( - blank=True, - help_text="The date the vulnerability was added to an exploit catalog.", - null=True, - ), - ), - ( - "description", - models.TextField( - blank=True, - help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", - null=True, - ), - ), - ( - "required_action", - models.TextField( - blank=True, - help_text="The required action to address the vulnerability, typically to apply vendor updates or apply vendor mitigations or to discontinue use.", - null=True, - ), - ), - ( - "due_date", - models.DateField( - blank=True, - help_text="The date the required action is due, which applies to all USA federal civilian executive branch (FCEB) agencies, but all organizations are strongly encouraged to execute the required action", - null=True, - ), - ), - ( - "notes", - models.TextField( - blank=True, - help_text="Additional notes and resources about the vulnerability, often a URL to vendor instructions.", - null=True, - ), - ), - ( - "known_ransomware_campaign_use", - models.BooleanField( - default=False, - help_text="Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; \n or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.", - ), - ), - ( - "source_date_published", - models.DateField( - blank=True, - help_text="The date that the exploit was published or disclosed.", - null=True, - ), - ), - ( - "exploit_type", - models.TextField( - blank=True, - help_text="The type of the exploit as provided by the original upstream data source.", - null=True, - ), - ), - ( - "platform", - models.TextField( - blank=True, - help_text="The platform associated with the exploit as provided by the original upstream data source.", - null=True, - ), - ), - ( - "source_date_updated", - models.DateField( - blank=True, - help_text="The date the exploit was updated in the original upstream data source.", - null=True, - ), - ), - ( - "data_source", - models.TextField( - blank=True, - help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", - null=True, - ), - ), - ( - "source_url", - models.URLField( - blank=True, - help_text="The URL to the exploit as provided in the original upstream data source.", - null=True, - ), - ), - ( - "vulnerability", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="exploits", - to="vulnerabilities.vulnerability", - ), - ), - ], - ), - migrations.DeleteModel( - name="Kev", - ), - ] From 2d9e5858396066feebf38f194cc1625f1f931a2e Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 17 Sep 2024 05:13:41 +0300 Subject: [PATCH 07/20] Add support for Calculating Risk in VulnerableCode Signed-off-by: ziadhany Signed-off-by: ziad hany --- vulnerabilities/risk.py | 125 +++++++++++++ .../templates/package_details.html | 10 ++ vulnerabilities/tests/test_risk.py | 164 ++++++++++++++++++ vulnerabilities/views.py | 2 + weight_config.json | 5 + 5 files changed, 306 insertions(+) create mode 100644 vulnerabilities/risk.py create mode 100644 vulnerabilities/tests/test_risk.py create mode 100644 weight_config.json diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py new file mode 100644 index 000000000..5be6e4ebf --- /dev/null +++ b/vulnerabilities/risk.py @@ -0,0 +1,125 @@ +import os +import re + +from vulnerabilities.models import Exploit +from vulnerabilities.models import Package +from vulnerabilities.models import PackageRelatedVulnerability +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.severity_systems import EPSS +from vulnerabilities.utils import load_json + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def get_weighted_severity(severities): + """ + Weighted Severity is the maximum value obtained when each Severity is multiplied + by its associated Weight/10. + Example of Weighted Severity: max(7*(10/10), 8*(3/10), 6*(8/10)) = 7 + """ + weight_config_path = os.path.join(BASE_DIR, "..", "weight_config.json") + weight_config = load_json(weight_config_path) + + score_map = { + "low": 3, + "moderate": 6.9, + "medium": 6.9, + "high": 8.9, + "important": 8.9, + "critical": 10.0, + "urgent": 10.0, + } + + score_list = [] + for severity in severities: + weights = [ + value + for regex_key, value in weight_config.items() + if re.match(regex_key, severity.reference.url) + ] + + if not weights: + return 0 + + max_weight = float(max(weights)) / 10 + vul_score = severity.value + try: + vul_score = float(vul_score) + vul_score_value = vul_score * max_weight + except ValueError: + vul_score = vul_score.lower() + vul_score_value = score_map.get(vul_score, 0) * max_weight + + score_list.append(vul_score_value) + return max(score_list) if score_list else 0 + + +def get_exploitability_level(exploits, references, severities): + """ + Exploitability refers to the potential or + probability of a software package vulnerability being exploited by + malicious actors to compromise systems, applications, or networks. + It is determined automatically by discovery of exploits. + """ + # no exploit known ( default .5) + exploit_level = 0.5 + + if exploits: + # Automatable Exploit with PoC script published OR known exploits (KEV) in the wild OR known ransomware + exploit_level = 2 + + elif severities: + # high EPSS. + epss = severities.filter( + scoring_system=EPSS.identifier, + ) + epss = any(float(epss.value) > 0.8 for epss in epss) + if epss: + exploit_level = 2 + + elif references: + # PoC/Exploit script published + ref_exploits = references.filter( + reference_type=VulnerabilityReference.EXPLOIT, + ) + if ref_exploits: + exploit_level = 1 + + return exploit_level + + +def calculate_vulnerability_risk(vulnerability: Vulnerability): + """ + Risk may be expressed as a number ranging from 0 to 10. + Risk is calculated from weighted severity and exploitability values. + It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 + + Risk = min(weighted severity * exploitability, 10) + """ + # TODO try to use a prefetch + references = vulnerability.references.all() + severities = vulnerability.severities.all() + exploits = Exploit.objects.filter(vulnerability=vulnerability) + + weighted_severity = get_weighted_severity(severities) + exploitability = get_exploitability_level(exploits, references, severities) + return min(weighted_severity * exploitability, 10) + + +def calculate_pkg_risk(package: Package): + """ + Calculate the risk for a package by iterating over all vulnerabilities that affects this package + and determining the associated risk. + """ + + result = [] + for pkg_related_vul in PackageRelatedVulnerability.objects.filter(package=package, fix=False): + if pkg_related_vul: + risk = calculate_vulnerability_risk(pkg_related_vul.vulnerability) + result.append(risk) + + if not result: + return + + return f"{max(result):.2f}" diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 26de42fa8..85cb10cdd 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -112,6 +112,16 @@ {% endif %} + + + Risk + + + {% if risk %} + {{ risk }} + {% endif %} + + diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py new file mode 100644 index 000000000..30e6e9807 --- /dev/null +++ b/vulnerabilities/tests/test_risk.py @@ -0,0 +1,164 @@ +import pytest + +from vulnerabilities.models import Exploit +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.models import VulnerabilitySeverity +from vulnerabilities.models import Weakness +from vulnerabilities.risk import calculate_vulnerability_risk +from vulnerabilities.risk import get_exploitability_level +from vulnerabilities.risk import get_weighted_severity +from vulnerabilities.severity_systems import CVSSV3 +from vulnerabilities.severity_systems import EPSS +from vulnerabilities.severity_systems import GENERIC + + +@pytest.fixture +@pytest.mark.django_db +def vulnerability(): + vul = Vulnerability(vulnerability_id="VCID-Existing") + vul.save() + + reference1 = VulnerabilityReference.objects.create( + reference_id="", + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx1", + ) + + VulnerabilitySeverity.objects.create( + reference=reference1, + scoring_system=CVSSV3.identifier, + scoring_elements="CVSS:3.0/AV:P/AC:H/PR:H/UI:R/S:C/C:H/I:H/A:N/E:H/RL:O/RC:R/CR:H/MAC:H/MC:L", + value="6.5", + ) + + VulnerabilitySeverity.objects.create( + reference=reference1, + scoring_system=GENERIC.identifier, + value="MODERATE", # 6.9 + ) + + VulnerabilityRelatedReference.objects.create(reference=reference1, vulnerability=vul) + + weaknesses = Weakness.objects.create(cwe_id=119) + vul.weaknesses.add(weaknesses) + return vul + + +@pytest.fixture +@pytest.mark.django_db +def exploit(): + vul = Vulnerability(vulnerability_id="VCID-Exploit") + vul.save() + return Exploit.objects.create(vulnerability=vul, description="exploit description") + + +@pytest.fixture +@pytest.mark.django_db +def vulnerability_with_exploit_ref(): + vul = Vulnerability(vulnerability_id="VCID-Exploit-Ref") + vul.save() + + reference_exploit = VulnerabilityReference.objects.create( + reference_id="", + reference_type=VulnerabilityReference.EXPLOIT, + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxxx2", + ) + + VulnerabilityRelatedReference.objects.create(reference=reference_exploit, vulnerability=vul) + return vul + + +@pytest.fixture +@pytest.mark.django_db +def high_epss_score(): + vul = Vulnerability(vulnerability_id="VCID-HIGH-EPSS") + vul.save() + + reference1 = VulnerabilityReference.objects.create( + reference_id="", + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx3", + ) + + VulnerabilitySeverity.objects.create( + reference=reference1, + scoring_system=EPSS.identifier, + value=".9", + ) + + VulnerabilityRelatedReference.objects.create(reference=reference1, vulnerability=vul) + return vul.severities + + +@pytest.fixture +@pytest.mark.django_db +def low_epss_score(): + vul = Vulnerability(vulnerability_id="VCID-LOW-EPSS") + vul.save() + + reference1 = VulnerabilityReference.objects.create( + reference_id="", + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx4", + ) + + VulnerabilitySeverity.objects.create( + reference=reference1, + scoring_system=EPSS.identifier, + value=".3", + ) + + VulnerabilityRelatedReference.objects.create(reference=reference1, vulnerability=vul) + return vul.severities + + +@pytest.mark.django_db +def test_exploitability_level( + exploit, + vulnerability_with_exploit_ref, + high_epss_score, + low_epss_score, + vulnerability, +): + + assert get_exploitability_level(exploit, None, None) == 2 + + assert get_exploitability_level(None, None, high_epss_score) == 2 + + assert get_exploitability_level(None, None, low_epss_score) == 0.5 + + assert ( + get_exploitability_level( + None, + vulnerability_with_exploit_ref.references, + vulnerability_with_exploit_ref.severities, + ) + == 1 + ) + + assert get_exploitability_level(None, None, None) == 0.5 + + +@pytest.mark.django_db +def test_get_weighted_severity(vulnerability): + severities = vulnerability.severities.all() + assert get_weighted_severity(severities) == 6.210000000000001 + + reference2 = VulnerabilityReference.objects.create( + reference_id="", + url="https://security-tracker.debian.org/tracker/CVE-2019-13057", + ) + + VulnerabilitySeverity.objects.create( + reference=reference2, + scoring_system=GENERIC.identifier, + value="CRITICAL", + ) + + VulnerabilityRelatedReference.objects.create(reference=reference2, vulnerability=vulnerability) + new_severities = vulnerability.severities.all() + assert get_weighted_severity(new_severities) == 9 + + +@pytest.mark.django_db +def test_calculate_vulnerability_risk(vulnerability): + assert calculate_vulnerability_risk(vulnerability) == 3.1050000000000004 diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 394dc1c36..75b510cdc 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -31,6 +31,7 @@ from vulnerabilities.forms import PackageSearchForm from vulnerabilities.forms import VulnerabilitySearchForm from vulnerabilities.models import VulnerabilityStatusType +from vulnerabilities.risk import calculate_pkg_risk from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import get_severity_range @@ -122,6 +123,7 @@ def get_context_data(self, **kwargs): context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) context["fixed_package_details"] = package.fixed_package_details + context["risk"] = calculate_pkg_risk(package) context["history"] = list(package.history) return context diff --git a/weight_config.json b/weight_config.json new file mode 100644 index 000000000..7dd69c4ae --- /dev/null +++ b/weight_config.json @@ -0,0 +1,5 @@ +{ + "https://nvd\\.nist\\.gov/.*": 9, + "https:\\/\\/security-tracker\\.debian\\.org\\/.*": 9, + "^(?:http|ftp)s?://": 1 +} \ No newline at end of file From 22cdb76405a155a605064bd0ff2e20d714f96e97 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 1 Oct 2024 03:16:09 +0300 Subject: [PATCH 08/20] Remove unwanted migration file Signed-off-by: ziadhany Signed-off-by: ziad hany --- .../migrations/0065_exploit_delete_kev.py | 131 ------------------ 1 file changed, 131 deletions(-) delete mode 100644 vulnerabilities/migrations/0065_exploit_delete_kev.py diff --git a/vulnerabilities/migrations/0065_exploit_delete_kev.py b/vulnerabilities/migrations/0065_exploit_delete_kev.py deleted file mode 100644 index 28b14d2c9..000000000 --- a/vulnerabilities/migrations/0065_exploit_delete_kev.py +++ /dev/null @@ -1,131 +0,0 @@ -# Generated by Django 4.2.15 on 2024-09-21 15:37 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ("vulnerabilities", "0064_update_npm_pypa_advisory_created_by"), - ] - - operations = [ - migrations.CreateModel( - name="Exploit", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, primary_key=True, serialize=False, verbose_name="ID" - ), - ), - ( - "date_added", - models.DateField( - blank=True, - help_text="The date the vulnerability was added to an exploit catalog.", - null=True, - ), - ), - ( - "description", - models.TextField( - blank=True, - help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", - null=True, - ), - ), - ( - "required_action", - models.TextField( - blank=True, - help_text="The required action to address the vulnerability, typically to apply vendor updates or apply vendor mitigations or to discontinue use.", - null=True, - ), - ), - ( - "due_date", - models.DateField( - blank=True, - help_text="The date the required action is due, which applies to all USA federal civilian executive branch (FCEB) agencies, but all organizations are strongly encouraged to execute the required action", - null=True, - ), - ), - ( - "notes", - models.TextField( - blank=True, - help_text="Additional notes and resources about the vulnerability, often a URL to vendor instructions.", - null=True, - ), - ), - ( - "known_ransomware_campaign_use", - models.BooleanField( - default=False, - help_text="Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; \n or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.", - ), - ), - ( - "source_date_published", - models.DateField( - blank=True, - help_text="The date that the exploit was published or disclosed.", - null=True, - ), - ), - ( - "exploit_type", - models.TextField( - blank=True, - help_text="The type of the exploit as provided by the original upstream data source.", - null=True, - ), - ), - ( - "platform", - models.TextField( - blank=True, - help_text="The platform associated with the exploit as provided by the original upstream data source.", - null=True, - ), - ), - ( - "source_date_updated", - models.DateField( - blank=True, - help_text="The date the exploit was updated in the original upstream data source.", - null=True, - ), - ), - ( - "data_source", - models.TextField( - blank=True, - help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", - null=True, - ), - ), - ( - "source_url", - models.URLField( - blank=True, - help_text="The URL to the exploit as provided in the original upstream data source.", - null=True, - ), - ), - ( - "vulnerability", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="exploits", - to="vulnerabilities.vulnerability", - ), - ), - ], - ), - migrations.DeleteModel( - name="Kev", - ), - ] From 8beb3b9ee8c7cec21e7d8968217a50c360fe89b6 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 1 Oct 2024 17:47:30 +0300 Subject: [PATCH 09/20] Add a prefetch to try to optimize query performance Signed-off-by: ziadhany Signed-off-by: ziad hany --- vulnerabilities/risk.py | 9 +++++---- vulnerabilities/views.py | 1 - 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index 5be6e4ebf..c6a772d40 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -97,9 +97,8 @@ def calculate_vulnerability_risk(vulnerability: Vulnerability): Risk = min(weighted severity * exploitability, 10) """ - # TODO try to use a prefetch - references = vulnerability.references.all() - severities = vulnerability.severities.all() + references = vulnerability.references.select_related("url", "reference_type") + severities = vulnerability.severities.select_related("reference") exploits = Exploit.objects.filter(vulnerability=vulnerability) weighted_severity = get_weighted_severity(severities) @@ -114,7 +113,9 @@ def calculate_pkg_risk(package: Package): """ result = [] - for pkg_related_vul in PackageRelatedVulnerability.objects.filter(package=package, fix=False): + for pkg_related_vul in PackageRelatedVulnerability.objects.filter( + package=package, fix=False + ).prefetch_related("vulnerability"): if pkg_related_vul: risk = calculate_vulnerability_risk(pkg_related_vul.vulnerability) result.append(risk) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 75b510cdc..30217919f 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -124,7 +124,6 @@ def get_context_data(self, **kwargs): context["package_search_form"] = PackageSearchForm(self.request.GET) context["fixed_package_details"] = package.fixed_package_details context["risk"] = calculate_pkg_risk(package) - context["history"] = list(package.history) return context From 74a16ffed760064bb2251bc8eca5b98265291b78 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 1 Oct 2024 18:58:26 +0300 Subject: [PATCH 10/20] Empty risk when there is no data Signed-off-by: ziadhany Signed-off-by: ziad hany --- vulnerabilities/risk.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index c6a772d40..d68a2b3e1 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -97,13 +97,13 @@ def calculate_vulnerability_risk(vulnerability: Vulnerability): Risk = min(weighted severity * exploitability, 10) """ - references = vulnerability.references.select_related("url", "reference_type") + references = vulnerability.references severities = vulnerability.severities.select_related("reference") exploits = Exploit.objects.filter(vulnerability=vulnerability) - - weighted_severity = get_weighted_severity(severities) - exploitability = get_exploitability_level(exploits, references, severities) - return min(weighted_severity * exploitability, 10) + if references.exists() or severities.exists() or exploits.exists(): + weighted_severity = get_weighted_severity(severities) + exploitability = get_exploitability_level(exploits, references, severities) + return min(weighted_severity * exploitability, 10) def calculate_pkg_risk(package: Package): @@ -118,6 +118,8 @@ def calculate_pkg_risk(package: Package): ).prefetch_related("vulnerability"): if pkg_related_vul: risk = calculate_vulnerability_risk(pkg_related_vul.vulnerability) + if not risk: + continue result.append(risk) if not result: From 8a637fc724f2c232beea5dd7fc0334663bf59443 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 22 Oct 2024 09:45:07 +0300 Subject: [PATCH 11/20] Create a pipeline for package risk Signed-off-by: ziadhany Signed-off-by: ziad hany --- vulnerabilities/api.py | 1 + vulnerabilities/improvers/__init__.py | 44 ++++++++++--------- .../migrations/0074_package_risk.py | 23 ++++++++++ vulnerabilities/models.py | 8 ++++ vulnerabilities/pipelines/risk_package.py | 30 +++++++++++++ vulnerabilities/risk.py | 33 +++++++------- .../templates/package_details.html | 4 +- .../tests/pipelines/test_risk_pipeline.py | 24 ++++++++++ vulnerabilities/tests/test_api.py | 1 + vulnerabilities/views.py | 1 - weight_config.json | 5 +-- 11 files changed, 131 insertions(+), 43 deletions(-) create mode 100644 vulnerabilities/migrations/0074_package_risk.py create mode 100644 vulnerabilities/pipelines/risk_package.py create mode 100644 vulnerabilities/tests/pipelines/test_risk_pipeline.py diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index e7fc33460..c4ca03447 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -359,6 +359,7 @@ class Meta: "latest_non_vulnerable_version", "affected_by_vulnerabilities", "fixing_vulnerabilities", + "risk", ] diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 6e9c24b38..f0ef42b98 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -14,29 +14,31 @@ from vulnerabilities.pipelines import enhance_with_kev from vulnerabilities.pipelines import enhance_with_metasploit from vulnerabilities.pipelines import flag_ghost_packages +from vulnerabilities.pipelines import risk_package IMPROVERS_REGISTRY = [ - valid_versions.GitHubBasicImprover, - valid_versions.GitLabBasicImprover, - valid_versions.NginxBasicImprover, - valid_versions.ApacheHTTPDImprover, - valid_versions.DebianBasicImprover, - valid_versions.NpmImprover, - valid_versions.ElixirImprover, - valid_versions.ApacheTomcatImprover, - valid_versions.ApacheKafkaImprover, - valid_versions.IstioImprover, - valid_versions.DebianOvalImprover, - valid_versions.UbuntuOvalImprover, - valid_versions.OSSFuzzImprover, - valid_versions.RubyImprover, - valid_versions.GithubOSVImprover, - vulnerability_status.VulnerabilityStatusImprover, - valid_versions.CurlImprover, - flag_ghost_packages.FlagGhostPackagePipeline, - enhance_with_kev.VulnerabilityKevPipeline, - enhance_with_metasploit.MetasploitImproverPipeline, - enhance_with_exploitdb.ExploitDBImproverPipeline, + # valid_versions.GitHubBasicImprover, + # valid_versions.GitLabBasicImprover, + # valid_versions.NginxBasicImprover, + # valid_versions.ApacheHTTPDImprover, + # valid_versions.DebianBasicImprover, + # valid_versions.NpmImprover, + # valid_versions.ElixirImprover, + # valid_versions.ApacheTomcatImprover, + # valid_versions.ApacheKafkaImprover, + # valid_versions.IstioImprover, + # valid_versions.DebianOvalImprover, + # valid_versions.UbuntuOvalImprover, + # valid_versions.OSSFuzzImprover, + # valid_versions.RubyImprover, + # valid_versions.GithubOSVImprover, + # vulnerability_status.VulnerabilityStatusImprover, + # valid_versions.CurlImprover, + # flag_ghost_packages.FlagGhostPackagePipeline, + # enhance_with_kev.VulnerabilityKevPipeline, + # enhance_with_metasploit.MetasploitImproverPipeline, + # enhance_with_exploitdb.ExploitDBImproverPipeline, + risk_package.RiskPackagePipeline, ] IMPROVERS_REGISTRY = { diff --git a/vulnerabilities/migrations/0074_package_risk.py b/vulnerabilities/migrations/0074_package_risk.py new file mode 100644 index 000000000..91ce9a0ea --- /dev/null +++ b/vulnerabilities/migrations/0074_package_risk.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.16 on 2024-10-22 06:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0073_delete_packagerelatedvulnerability"), + ] + + operations = [ + migrations.AddField( + model_name="package", + name="risk", + field=models.DecimalField( + decimal_places=2, + help_text="Enter a risk score between 0.00 and 10.00, where higher values indicate greater vulnerability risk for the package.", + max_digits=4, + null=True, + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 03ee82d1f..7726bfa79 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -636,6 +636,14 @@ class Package(PackageURLMixin): help_text="True if the package does not exist in the upstream package manager or its repository.", ) + risk = models.DecimalField( + null=True, + max_digits=4, + decimal_places=2, + help_text="Enter a risk score between 0.00 and 10.00, where higher values " + "indicate greater vulnerability risk for the package.", + ) + objects = PackageQuerySet.as_manager() def save(self, *args, **kwargs): diff --git a/vulnerabilities/pipelines/risk_package.py b/vulnerabilities/pipelines/risk_package.py new file mode 100644 index 000000000..95db46d2a --- /dev/null +++ b/vulnerabilities/pipelines/risk_package.py @@ -0,0 +1,30 @@ +from vulnerabilities.models import Package +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.risk import calculate_pkg_risk + + +class RiskPackagePipeline(VulnerableCodePipeline): + """ + Risk Assessment Pipeline for Package Vulnerabilities: Iterate through the packages and evaluate their associated risk. + """ + + pipeline_id = "risk_package" + license_expression = None + + @classmethod + def steps(cls): + return (cls.add_risk_package,) + + def add_risk_package(self): + self.log(f"Add risk package pipeline ") + + updatables = [] + for pkg in Package.objects.filter(affected_by_vulnerabilities__isnull=False): + risk = calculate_pkg_risk(pkg) + pkg.risk = risk + updatables.append(pkg) + + # Bulk update the 'risk' field for all packages + Package.objects.bulk_update(objs=updatables, fields=["risk"], batch_size=1000) + + self.log(f"Successfully added risk package pipeline ") diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index d68a2b3e1..de00132eb 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -1,15 +1,16 @@ import os -import re +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Exploit from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.severity_systems import EPSS from vulnerabilities.utils import load_json BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +WEIGHT_CONFIG_PATH = os.path.join(BASE_DIR, "../weight_config.json") +DEFAULT_WEIGHT = 1 def get_weighted_severity(severities): @@ -18,8 +19,8 @@ def get_weighted_severity(severities): by its associated Weight/10. Example of Weighted Severity: max(7*(10/10), 8*(3/10), 6*(8/10)) = 7 """ - weight_config_path = os.path.join(BASE_DIR, "..", "weight_config.json") - weight_config = load_json(weight_config_path) + + weight_config = load_json(WEIGHT_CONFIG_PATH) score_map = { "low": 3, @@ -33,11 +34,12 @@ def get_weighted_severity(severities): score_list = [] for severity in severities: - weights = [ - value - for regex_key, value in weight_config.items() - if re.match(regex_key, severity.reference.url) - ] + weights = [] + for key, value in weight_config.items(): + if severity.reference.url.startswith(key): + weights.append(value) + continue + weights.append(DEFAULT_WEIGHT) if not weights: return 0 @@ -113,14 +115,13 @@ def calculate_pkg_risk(package: Package): """ result = [] - for pkg_related_vul in PackageRelatedVulnerability.objects.filter( - package=package, fix=False + for pkg_related_vul in AffectedByPackageRelatedVulnerability.objects.filter( + package=package ).prefetch_related("vulnerability"): - if pkg_related_vul: - risk = calculate_vulnerability_risk(pkg_related_vul.vulnerability) - if not risk: - continue - result.append(risk) + risk = calculate_vulnerability_risk(pkg_related_vul.vulnerability) + if not risk: + continue + result.append(risk) if not result: return diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 85cb10cdd..ee7323287 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -117,8 +117,8 @@ Risk - {% if risk %} - {{ risk }} + {% if package.risk %} + {{ package.risk }} {% endif %} diff --git a/vulnerabilities/tests/pipelines/test_risk_pipeline.py b/vulnerabilities/tests/pipelines/test_risk_pipeline.py new file mode 100644 index 000000000..f65b8910a --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_risk_pipeline.py @@ -0,0 +1,24 @@ +import pytest + +from vulnerabilities.models import AffectedByPackageRelatedVulnerability +from vulnerabilities.models import Package +from vulnerabilities.pipelines.risk_package import RiskPackagePipeline +from vulnerabilities.tests.test_risk import vulnerability + + +@pytest.mark.django_db +def test_simple_risk_pipeline(vulnerability): + pkg = Package.objects.create(type="pypi", name="foo", version="2.3.0") + assert Package.objects.count() == 1 + + improver = RiskPackagePipeline() + improver.execute() + + assert pkg.risk is None + + AffectedByPackageRelatedVulnerability.objects.create(package=pkg, vulnerability=vulnerability) + improver = RiskPackagePipeline() + improver.execute() + + pkg = Package.objects.get(type="pypi", name="foo", version="2.3.0") + assert str(pkg.risk) == str(3.11) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index cbb018673..9a982e0b4 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -625,6 +625,7 @@ def test_api_with_lesser_and_greater_fixed_by_packages(self): } ], "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "risk": None, } assert response == expected diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 30217919f..d0a58effe 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -123,7 +123,6 @@ def get_context_data(self, **kwargs): context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) context["fixed_package_details"] = package.fixed_package_details - context["risk"] = calculate_pkg_risk(package) context["history"] = list(package.history) return context diff --git a/weight_config.json b/weight_config.json index 7dd69c4ae..8951dad03 100644 --- a/weight_config.json +++ b/weight_config.json @@ -1,5 +1,4 @@ { - "https://nvd\\.nist\\.gov/.*": 9, - "https:\\/\\/security-tracker\\.debian\\.org\\/.*": 9, - "^(?:http|ftp)s?://": 1 + "https://nvd.nist.gov/": 9, + "https://security-tracker.debian.org/": 9 } \ No newline at end of file From 848a5f56c5791b767e97878f4a3b9a8d536a8a92 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 22 Oct 2024 16:23:21 +0300 Subject: [PATCH 12/20] Load the weight once uncomment all importers Signed-off-by: ziad hany --- vulnerabilities/improvers/__init__.py | 42 +++++++++++++-------------- vulnerabilities/risk.py | 5 ++-- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index f0ef42b98..1885473bf 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -17,27 +17,27 @@ from vulnerabilities.pipelines import risk_package IMPROVERS_REGISTRY = [ - # valid_versions.GitHubBasicImprover, - # valid_versions.GitLabBasicImprover, - # valid_versions.NginxBasicImprover, - # valid_versions.ApacheHTTPDImprover, - # valid_versions.DebianBasicImprover, - # valid_versions.NpmImprover, - # valid_versions.ElixirImprover, - # valid_versions.ApacheTomcatImprover, - # valid_versions.ApacheKafkaImprover, - # valid_versions.IstioImprover, - # valid_versions.DebianOvalImprover, - # valid_versions.UbuntuOvalImprover, - # valid_versions.OSSFuzzImprover, - # valid_versions.RubyImprover, - # valid_versions.GithubOSVImprover, - # vulnerability_status.VulnerabilityStatusImprover, - # valid_versions.CurlImprover, - # flag_ghost_packages.FlagGhostPackagePipeline, - # enhance_with_kev.VulnerabilityKevPipeline, - # enhance_with_metasploit.MetasploitImproverPipeline, - # enhance_with_exploitdb.ExploitDBImproverPipeline, + valid_versions.GitHubBasicImprover, + valid_versions.GitLabBasicImprover, + valid_versions.NginxBasicImprover, + valid_versions.ApacheHTTPDImprover, + valid_versions.DebianBasicImprover, + valid_versions.NpmImprover, + valid_versions.ElixirImprover, + valid_versions.ApacheTomcatImprover, + valid_versions.ApacheKafkaImprover, + valid_versions.IstioImprover, + valid_versions.DebianOvalImprover, + valid_versions.UbuntuOvalImprover, + valid_versions.OSSFuzzImprover, + valid_versions.RubyImprover, + valid_versions.GithubOSVImprover, + vulnerability_status.VulnerabilityStatusImprover, + valid_versions.CurlImprover, + flag_ghost_packages.FlagGhostPackagePipeline, + enhance_with_kev.VulnerabilityKevPipeline, + enhance_with_metasploit.MetasploitImproverPipeline, + enhance_with_exploitdb.ExploitDBImproverPipeline, risk_package.RiskPackagePipeline, ] diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index de00132eb..ba656e4ba 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -11,6 +11,7 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) WEIGHT_CONFIG_PATH = os.path.join(BASE_DIR, "../weight_config.json") DEFAULT_WEIGHT = 1 +WEIGHT_CONFIG = load_json(WEIGHT_CONFIG_PATH) def get_weighted_severity(severities): @@ -20,8 +21,6 @@ def get_weighted_severity(severities): Example of Weighted Severity: max(7*(10/10), 8*(3/10), 6*(8/10)) = 7 """ - weight_config = load_json(WEIGHT_CONFIG_PATH) - score_map = { "low": 3, "moderate": 6.9, @@ -35,7 +34,7 @@ def get_weighted_severity(severities): score_list = [] for severity in severities: weights = [] - for key, value in weight_config.items(): + for key, value in WEIGHT_CONFIG.items(): if severity.reference.url.startswith(key): weights.append(value) continue From 185eb0e5bda8fceebbe8be4e9200762bcae928ee Mon Sep 17 00:00:00 2001 From: ziadhany Date: Mon, 28 Oct 2024 17:46:40 +0300 Subject: [PATCH 13/20] Update the risk description in the model. Rename the pipeline from RiskPackagePipeline to ComputePackageRiskPipeline. Add a tooltip for risk, and remove any unused imports in the view. Signed-off-by: ziad hany --- vulnerabilities/improvers/__init__.py | 2 +- .../migrations/0074_package_risk.py | 4 +-- vulnerabilities/models.py | 2 +- vulnerabilities/pipelines/risk_package.py | 36 +++++++++++++++---- vulnerabilities/risk.py | 11 +++--- .../templates/package_details.html | 6 +++- .../tests/pipelines/test_risk_pipeline.py | 6 ++-- vulnerabilities/views.py | 1 - 8 files changed, 45 insertions(+), 23 deletions(-) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 1885473bf..0f14f4854 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -38,7 +38,7 @@ enhance_with_kev.VulnerabilityKevPipeline, enhance_with_metasploit.MetasploitImproverPipeline, enhance_with_exploitdb.ExploitDBImproverPipeline, - risk_package.RiskPackagePipeline, + risk_package.ComputePackageRiskPipeline, ] IMPROVERS_REGISTRY = { diff --git a/vulnerabilities/migrations/0074_package_risk.py b/vulnerabilities/migrations/0074_package_risk.py index 91ce9a0ea..6dd8f5db0 100644 --- a/vulnerabilities/migrations/0074_package_risk.py +++ b/vulnerabilities/migrations/0074_package_risk.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.16 on 2024-10-22 06:49 +# Generated by Django 4.2.16 on 2024-10-28 13:02 from django.db import migrations, models @@ -15,7 +15,7 @@ class Migration(migrations.Migration): name="risk", field=models.DecimalField( decimal_places=2, - help_text="Enter a risk score between 0.00 and 10.00, where higher values indicate greater vulnerability risk for the package.", + help_text="Risk score between 0.00 and 10.00, where higher values indicate greater vulnerability risk for the package.", max_digits=4, null=True, ), diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 7726bfa79..21f1a94c0 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -640,7 +640,7 @@ class Package(PackageURLMixin): null=True, max_digits=4, decimal_places=2, - help_text="Enter a risk score between 0.00 and 10.00, where higher values " + help_text="Risk score between 0.00 and 10.00, where higher values " "indicate greater vulnerability risk for the package.", ) diff --git a/vulnerabilities/pipelines/risk_package.py b/vulnerabilities/pipelines/risk_package.py index 95db46d2a..0b05655de 100644 --- a/vulnerabilities/pipelines/risk_package.py +++ b/vulnerabilities/pipelines/risk_package.py @@ -1,14 +1,16 @@ +from aboutcode.pipeline import LoopProgress + from vulnerabilities.models import Package from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.risk import calculate_pkg_risk -class RiskPackagePipeline(VulnerableCodePipeline): +class ComputePackageRiskPipeline(VulnerableCodePipeline): """ Risk Assessment Pipeline for Package Vulnerabilities: Iterate through the packages and evaluate their associated risk. """ - pipeline_id = "risk_package" + pipeline_id = "compute_package_risk" license_expression = None @classmethod @@ -16,15 +18,35 @@ def steps(cls): return (cls.add_risk_package,) def add_risk_package(self): - self.log(f"Add risk package pipeline ") + affected_pkgs = Package.objects.filter(affected_by_vulnerabilities__isnull=False).distinct() + + self.log(f"Calculating risk for {affected_pkgs.count():,d} affected package records") + + progress = LoopProgress(total_iterations=affected_pkgs.count(), logger=self.log) updatables = [] - for pkg in Package.objects.filter(affected_by_vulnerabilities__isnull=False): + updated_package_count = 0 + batch_size = 1000 + + for pkg in progress.iter(affected_pkgs): risk = calculate_pkg_risk(pkg) pkg.risk = risk updatables.append(pkg) - # Bulk update the 'risk' field for all packages - Package.objects.bulk_update(objs=updatables, fields=["risk"], batch_size=1000) + if len(updatables) >= batch_size: + try: + Package.objects.bulk_update(objs=updatables, fields=["risk"]) + updated_package_count += len(updatables) + except Exception as e: + self.log(f"Error updating packages: {e}") + + updatables.clear() + + if updatables: + try: + Package.objects.bulk_update(objs=updatables, fields=["risk"]) + updated_package_count += len(updatables) + except Exception as e: + self.log(f"Error updating remaining packages: {e}") - self.log(f"Successfully added risk package pipeline ") + self.log(f"Successfully added risk score for {updated_package_count:,d} package") diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index ba656e4ba..df38b0372 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Exploit @@ -8,9 +8,8 @@ from vulnerabilities.severity_systems import EPSS from vulnerabilities.utils import load_json -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -WEIGHT_CONFIG_PATH = os.path.join(BASE_DIR, "../weight_config.json") DEFAULT_WEIGHT = 1 +WEIGHT_CONFIG_PATH = Path(__file__).parent.parent / "weight_config.json" WEIGHT_CONFIG = load_json(WEIGHT_CONFIG_PATH) @@ -117,10 +116,8 @@ def calculate_pkg_risk(package: Package): for pkg_related_vul in AffectedByPackageRelatedVulnerability.objects.filter( package=package ).prefetch_related("vulnerability"): - risk = calculate_vulnerability_risk(pkg_related_vul.vulnerability) - if not risk: - continue - result.append(risk) + if risk := calculate_vulnerability_risk(pkg_related_vul.vulnerability): + result.append(risk) if not result: return diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index ee7323287..20480b943 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -114,7 +114,11 @@ - Risk + + Risk + {% if package.risk %} diff --git a/vulnerabilities/tests/pipelines/test_risk_pipeline.py b/vulnerabilities/tests/pipelines/test_risk_pipeline.py index f65b8910a..b69cb626a 100644 --- a/vulnerabilities/tests/pipelines/test_risk_pipeline.py +++ b/vulnerabilities/tests/pipelines/test_risk_pipeline.py @@ -2,7 +2,7 @@ from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.pipelines.risk_package import RiskPackagePipeline +from vulnerabilities.pipelines.risk_package import ComputePackageRiskPipeline from vulnerabilities.tests.test_risk import vulnerability @@ -11,13 +11,13 @@ def test_simple_risk_pipeline(vulnerability): pkg = Package.objects.create(type="pypi", name="foo", version="2.3.0") assert Package.objects.count() == 1 - improver = RiskPackagePipeline() + improver = ComputePackageRiskPipeline() improver.execute() assert pkg.risk is None AffectedByPackageRelatedVulnerability.objects.create(package=pkg, vulnerability=vulnerability) - improver = RiskPackagePipeline() + improver = ComputePackageRiskPipeline() improver.execute() pkg = Package.objects.get(type="pypi", name="foo", version="2.3.0") diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index d0a58effe..d5e568701 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -31,7 +31,6 @@ from vulnerabilities.forms import PackageSearchForm from vulnerabilities.forms import VulnerabilitySearchForm from vulnerabilities.models import VulnerabilityStatusType -from vulnerabilities.risk import calculate_pkg_risk from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import get_severity_range From 45c62b5051625d82a95cef3443b14572597fd055 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Mon, 28 Oct 2024 18:01:03 +0300 Subject: [PATCH 14/20] Rename the pipeline step from add_risk_package to add_package_risk_score and remove any extra whitespace in views.py. Signed-off-by: ziad hany --- vulnerabilities/pipelines/risk_package.py | 4 ++-- vulnerabilities/views.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/pipelines/risk_package.py b/vulnerabilities/pipelines/risk_package.py index 0b05655de..46d5f12ba 100644 --- a/vulnerabilities/pipelines/risk_package.py +++ b/vulnerabilities/pipelines/risk_package.py @@ -15,9 +15,9 @@ class ComputePackageRiskPipeline(VulnerableCodePipeline): @classmethod def steps(cls): - return (cls.add_risk_package,) + return (cls.add_package_risk_score,) - def add_risk_package(self): + def add_package_risk_score(self): affected_pkgs = Package.objects.filter(affected_by_vulnerabilities__isnull=False).distinct() self.log(f"Calculating risk for {affected_pkgs.count():,d} affected package records") diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index e1b259d91..51cdcd049 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -122,6 +122,7 @@ def get_context_data(self, **kwargs): context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) context["fixed_package_details"] = package.fixed_package_details + context["history"] = list(package.history) return context From 0af9432bb0114f9c8c6ae96c774c53dd1e52ef6b Mon Sep 17 00:00:00 2001 From: ziadhany Date: Mon, 28 Oct 2024 18:03:52 +0300 Subject: [PATCH 15/20] Resolve migration conflict Signed-off-by: ziad hany --- .../migrations/{0074_package_risk.py => 0075_package_risk.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename vulnerabilities/migrations/{0074_package_risk.py => 0075_package_risk.py} (81%) diff --git a/vulnerabilities/migrations/0074_package_risk.py b/vulnerabilities/migrations/0075_package_risk.py similarity index 81% rename from vulnerabilities/migrations/0074_package_risk.py rename to vulnerabilities/migrations/0075_package_risk.py index 6dd8f5db0..5ffe849ef 100644 --- a/vulnerabilities/migrations/0074_package_risk.py +++ b/vulnerabilities/migrations/0075_package_risk.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.16 on 2024-10-28 13:02 +# Generated by Django 4.2.16 on 2024-10-28 15:02 from django.db import migrations, models @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ("vulnerabilities", "0073_delete_packagerelatedvulnerability"), + ("vulnerabilities", "0074_update_pysec_advisory_created_by"), ] operations = [ From 0e114ac059e1cb768e19a922c6bbd14f31643f14 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Tue, 29 Oct 2024 14:47:52 +0300 Subject: [PATCH 16/20] Rename the pipeline file Add pagination and refactor bulk_update_package Signed-off-by: ziad hany --- vulnerabilities/api.py | 2 +- vulnerabilities/improvers/__init__.py | 4 +- ...age_risk.py => 0075_package_risk_score.py} | 4 +- vulnerabilities/models.py | 2 +- .../pipelines/compute_package_risk.py | 59 +++++++++++++++++++ vulnerabilities/pipelines/risk_package.py | 52 ---------------- vulnerabilities/risk.py | 6 +- .../templates/package_details.html | 4 +- ...peline.py => test_compute_package_risk.py} | 6 +- vulnerabilities/tests/test_api.py | 2 +- vulnerabilities/tests/test_risk.py | 6 +- 11 files changed, 77 insertions(+), 70 deletions(-) rename vulnerabilities/migrations/{0075_package_risk.py => 0075_package_risk_score.py} (87%) create mode 100644 vulnerabilities/pipelines/compute_package_risk.py delete mode 100644 vulnerabilities/pipelines/risk_package.py rename vulnerabilities/tests/pipelines/{test_risk_pipeline.py => test_compute_package_risk.py} (80%) diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index c285055b2..b8bb703a6 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -359,7 +359,7 @@ class Meta: "latest_non_vulnerable_version", "affected_by_vulnerabilities", "fixing_vulnerabilities", - "risk", + "risk_score", ] diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 7241055eb..fd18fb28c 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -10,11 +10,11 @@ from vulnerabilities.improvers import valid_versions from vulnerabilities.improvers import vulnerability_status from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.pipelines import compute_package_risk from vulnerabilities.pipelines import enhance_with_exploitdb from vulnerabilities.pipelines import enhance_with_kev from vulnerabilities.pipelines import enhance_with_metasploit from vulnerabilities.pipelines import flag_ghost_packages -from vulnerabilities.pipelines import risk_package IMPROVERS_REGISTRY = [ valid_versions.GitHubBasicImprover, @@ -38,7 +38,7 @@ enhance_with_kev.VulnerabilityKevPipeline, enhance_with_metasploit.MetasploitImproverPipeline, enhance_with_exploitdb.ExploitDBImproverPipeline, - risk_package.ComputePackageRiskPipeline, + compute_package_risk.ComputePackageRiskPipeline, ] IMPROVERS_REGISTRY = { diff --git a/vulnerabilities/migrations/0075_package_risk.py b/vulnerabilities/migrations/0075_package_risk_score.py similarity index 87% rename from vulnerabilities/migrations/0075_package_risk.py rename to vulnerabilities/migrations/0075_package_risk_score.py index 5ffe849ef..72827ae63 100644 --- a/vulnerabilities/migrations/0075_package_risk.py +++ b/vulnerabilities/migrations/0075_package_risk_score.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.16 on 2024-10-28 15:02 +# Generated by Django 4.2.16 on 2024-10-29 10:55 from django.db import migrations, models @@ -12,7 +12,7 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( model_name="package", - name="risk", + name="risk_score", field=models.DecimalField( decimal_places=2, help_text="Risk score between 0.00 and 10.00, where higher values indicate greater vulnerability risk for the package.", diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index e410121ec..b95a07297 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -636,7 +636,7 @@ class Package(PackageURLMixin): help_text="True if the package does not exist in the upstream package manager or its repository.", ) - risk = models.DecimalField( + risk_score = models.DecimalField( null=True, max_digits=4, decimal_places=2, diff --git a/vulnerabilities/pipelines/compute_package_risk.py b/vulnerabilities/pipelines/compute_package_risk.py new file mode 100644 index 000000000..525da599d --- /dev/null +++ b/vulnerabilities/pipelines/compute_package_risk.py @@ -0,0 +1,59 @@ +from aboutcode.pipeline import LoopProgress + +from vulnerabilities.models import Package +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.risk import compute_package_risk + + +class ComputePackageRiskPipeline(VulnerableCodePipeline): + """ + Risk Assessment Pipeline for Package Vulnerabilities: Iterate through the packages and evaluate their associated risk. + """ + + pipeline_id = "compute_package_risk" + license_expression = None + + @classmethod + def steps(cls): + return (cls.add_package_risk_score,) + + def add_package_risk_score(self): + affected_packages = Package.objects.filter( + affected_by_vulnerabilities__isnull=False + ).distinct() + + self.log(f"Calculating risk for {affected_packages.count():,d} affected package records") + + progress = LoopProgress(total_iterations=affected_packages.count(), logger=self.log) + + updatables = [] + updated_package_count = 0 + batch_size = 5000 + + for package in progress.iter(affected_packages.paginated()): + risk_score = compute_package_risk(package) + package.risk_score = risk_score + updatables.append(package) + + if len(updatables) >= batch_size: + updated_package_count += bulk_update_package_risk_score( + packages=updatables, + logger=self.log, + ) + updated_package_count += bulk_update_package_risk_score( + packages=updatables, + logger=self.log, + ) + self.log(f"Successfully added risk score for {updated_package_count:,d} package") + + +def bulk_update_package_risk_score(packages, logger): + package_count = 0 + if packages: + try: + Package.objects.bulk_update(objs=packages, fields=["risk_score"]) + package_count += len(packages) + except Exception as e: + logger(f"Error updating packages: {e}") + packages.clear() + return package_count diff --git a/vulnerabilities/pipelines/risk_package.py b/vulnerabilities/pipelines/risk_package.py deleted file mode 100644 index 46d5f12ba..000000000 --- a/vulnerabilities/pipelines/risk_package.py +++ /dev/null @@ -1,52 +0,0 @@ -from aboutcode.pipeline import LoopProgress - -from vulnerabilities.models import Package -from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.risk import calculate_pkg_risk - - -class ComputePackageRiskPipeline(VulnerableCodePipeline): - """ - Risk Assessment Pipeline for Package Vulnerabilities: Iterate through the packages and evaluate their associated risk. - """ - - pipeline_id = "compute_package_risk" - license_expression = None - - @classmethod - def steps(cls): - return (cls.add_package_risk_score,) - - def add_package_risk_score(self): - affected_pkgs = Package.objects.filter(affected_by_vulnerabilities__isnull=False).distinct() - - self.log(f"Calculating risk for {affected_pkgs.count():,d} affected package records") - - progress = LoopProgress(total_iterations=affected_pkgs.count(), logger=self.log) - - updatables = [] - updated_package_count = 0 - batch_size = 1000 - - for pkg in progress.iter(affected_pkgs): - risk = calculate_pkg_risk(pkg) - pkg.risk = risk - updatables.append(pkg) - - if len(updatables) >= batch_size: - try: - Package.objects.bulk_update(objs=updatables, fields=["risk"]) - updated_package_count += len(updatables) - except Exception as e: - self.log(f"Error updating packages: {e}") - - updatables.clear() - - if updatables: - try: - Package.objects.bulk_update(objs=updatables, fields=["risk"]) - updated_package_count += len(updatables) - except Exception as e: - self.log(f"Error updating remaining packages: {e}") - - self.log(f"Successfully added risk score for {updated_package_count:,d} package") diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index df38b0372..f3b225189 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -89,7 +89,7 @@ def get_exploitability_level(exploits, references, severities): return exploit_level -def calculate_vulnerability_risk(vulnerability: Vulnerability): +def compute_vulnerability_risk(vulnerability: Vulnerability): """ Risk may be expressed as a number ranging from 0 to 10. Risk is calculated from weighted severity and exploitability values. @@ -106,7 +106,7 @@ def calculate_vulnerability_risk(vulnerability: Vulnerability): return min(weighted_severity * exploitability, 10) -def calculate_pkg_risk(package: Package): +def compute_package_risk(package: Package): """ Calculate the risk for a package by iterating over all vulnerabilities that affects this package and determining the associated risk. @@ -116,7 +116,7 @@ def calculate_pkg_risk(package: Package): for pkg_related_vul in AffectedByPackageRelatedVulnerability.objects.filter( package=package ).prefetch_related("vulnerability"): - if risk := calculate_vulnerability_risk(pkg_related_vul.vulnerability): + if risk := compute_vulnerability_risk(pkg_related_vul.vulnerability): result.append(risk) if not result: diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 20480b943..dd72d0500 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -121,8 +121,8 @@ - {% if package.risk %} - {{ package.risk }} + {% if package.risk_score %} + {{ package.risk_score }} {% endif %} diff --git a/vulnerabilities/tests/pipelines/test_risk_pipeline.py b/vulnerabilities/tests/pipelines/test_compute_package_risk.py similarity index 80% rename from vulnerabilities/tests/pipelines/test_risk_pipeline.py rename to vulnerabilities/tests/pipelines/test_compute_package_risk.py index b69cb626a..e44fa424a 100644 --- a/vulnerabilities/tests/pipelines/test_risk_pipeline.py +++ b/vulnerabilities/tests/pipelines/test_compute_package_risk.py @@ -2,7 +2,7 @@ from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.pipelines.risk_package import ComputePackageRiskPipeline +from vulnerabilities.pipelines.compute_package_risk import ComputePackageRiskPipeline from vulnerabilities.tests.test_risk import vulnerability @@ -14,11 +14,11 @@ def test_simple_risk_pipeline(vulnerability): improver = ComputePackageRiskPipeline() improver.execute() - assert pkg.risk is None + assert pkg.risk_score is None AffectedByPackageRelatedVulnerability.objects.create(package=pkg, vulnerability=vulnerability) improver = ComputePackageRiskPipeline() improver.execute() pkg = Package.objects.get(type="pypi", name="foo", version="2.3.0") - assert str(pkg.risk) == str(3.11) + assert str(pkg.risk_score) == str(3.11) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index b59b2cbcd..926d3c219 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -625,7 +625,7 @@ def test_api_with_lesser_and_greater_fixed_by_packages(self): } ], "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", - "risk": None, + "risk_score": None, } assert response == expected diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py index 30e6e9807..169024eb6 100644 --- a/vulnerabilities/tests/test_risk.py +++ b/vulnerabilities/tests/test_risk.py @@ -6,7 +6,7 @@ from vulnerabilities.models import VulnerabilityRelatedReference from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness -from vulnerabilities.risk import calculate_vulnerability_risk +from vulnerabilities.risk import compute_vulnerability_risk from vulnerabilities.risk import get_exploitability_level from vulnerabilities.risk import get_weighted_severity from vulnerabilities.severity_systems import CVSSV3 @@ -160,5 +160,5 @@ def test_get_weighted_severity(vulnerability): @pytest.mark.django_db -def test_calculate_vulnerability_risk(vulnerability): - assert calculate_vulnerability_risk(vulnerability) == 3.1050000000000004 +def test_compute_vulnerability_risk(vulnerability): + assert compute_vulnerability_risk(vulnerability) == 3.1050000000000004 From a121f444380907bdbaaae6ae031da0227b63d9e9 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Wed, 6 Nov 2024 20:37:00 +0200 Subject: [PATCH 17/20] Update the weight_config dict and modify it to use domain names. Signed-off-by: ziad hany --- vulnerabilities/risk.py | 18 +- vulnerabilities/tests/test_risk.py | 2 +- weight_config.json | 2910 +++++++++++++++++++++++++++- 3 files changed, 2915 insertions(+), 15 deletions(-) diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index f3b225189..815435136 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -1,4 +1,5 @@ from pathlib import Path +from urllib.parse import urlparse from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Exploit @@ -8,7 +9,7 @@ from vulnerabilities.severity_systems import EPSS from vulnerabilities.utils import load_json -DEFAULT_WEIGHT = 1 +DEFAULT_WEIGHT = 5 WEIGHT_CONFIG_PATH = Path(__file__).parent.parent / "weight_config.json" WEIGHT_CONFIG = load_json(WEIGHT_CONFIG_PATH) @@ -32,17 +33,10 @@ def get_weighted_severity(severities): score_list = [] for severity in severities: - weights = [] - for key, value in WEIGHT_CONFIG.items(): - if severity.reference.url.startswith(key): - weights.append(value) - continue - weights.append(DEFAULT_WEIGHT) - - if not weights: - return 0 - - max_weight = float(max(weights)) / 10 + parsed_url = urlparse(severity.reference.url) + severity_source = parsed_url.netloc.replace("www.", "", 1) + weight = WEIGHT_CONFIG.get(severity_source, DEFAULT_WEIGHT) + max_weight = float(weight) / 10 vul_score = severity.value try: vul_score = float(vul_score) diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py index 169024eb6..34fc5f397 100644 --- a/vulnerabilities/tests/test_risk.py +++ b/vulnerabilities/tests/test_risk.py @@ -156,7 +156,7 @@ def test_get_weighted_severity(vulnerability): VulnerabilityRelatedReference.objects.create(reference=reference2, vulnerability=vulnerability) new_severities = vulnerability.severities.all() - assert get_weighted_severity(new_severities) == 9 + assert get_weighted_severity(new_severities) == 7 @pytest.mark.django_db diff --git a/weight_config.json b/weight_config.json index 8951dad03..f6cd83349 100644 --- a/weight_config.json +++ b/weight_config.json @@ -1,4 +1,2910 @@ { - "https://nvd.nist.gov/": 9, - "https://security-tracker.debian.org/": 9 + "nvd.nist.gov": 9, + "api.first.org": 9, + "github.com": 9, + "access.redhat.com": 9, + "bugzilla.redhat.com": 9, + "cve.mitre.org": 9, + "people.canonical.com": 9, + "lists.apache.org": 9, + "ubuntu.com": 9, + "openwall.com": 9, + "lists.fedoraproject.org": 9, + "npmjs.com": 9, + "web.archive.org": 9, + "security.archlinux.org": 9, + "mozilla.org": 9, + "huntr.dev": 9, + "snyk.io": 9, + "vuldb.com": 9, + "jenkins.io": 9, + "security.netapp.com": 9, + "securityfocus.com": 9, + "lists.debian.org": 9, + "rustsec.org": 8, + "usn.ubuntu.com": 8, + "packetstormsecurity.com": 8, + "pkg.go.dev": 8, + "hackerone.com": 8, + "git.kernel.org": 8, + "groups.google.com": 8, + "security.snyk.io": 8, + "lists.opensuse.org": 8, + "rhn.redhat.com": 8, + "seclists.org": 8, + "code.google.com": 8, + "moodle.org": 8, + "portal.msrc.microsoft.com": 8, + "issues.apache.org": 7, + "debian.org": 7, + "jira.xwiki.org": 7, + "exchange.xforce.ibmcloud.com": 7, + "typo3.org": 7, + "exploit-db.com": 7, + "gist.github.com": 7, + "plugins.trac.wordpress.org": 7, + "bugs.wireshark.org": 7, + "git.moodle.org": 7, + "secunia.com": 7, + "huntr.com": 7, + "security.gentoo.org": 7, + "bugs.launchpad.net": 7, + "code.wireshark.org": 7, + "svn.apache.org": 7, + "httpd.apache.org": 7, + "securitytracker.com": 7, + "wordfence.com": 7, + "wireshark.org": 7, + "bugs.chromium.org": 7, + "gitlab.com": 7, + "silverstripe.org": 7, + "crbug.com": 7, + "phpmyadmin.net": 7, + "securitylab.github.com": 7, + "curl.se": 7, + "sourceforge.net": 7, + "drupal.org": 7, + "discuss.hashicorp.com": 7, + "pypi.org": 7, + "djangoproject.com": 7, + "bugzilla.mozilla.org": 7, + "raw.githubusercontent.com": 7, + "lists.gnu.org": 7, + "bugs.debian.org": 7, + "security-tracker.debian.org": 7, + "marc.info": 7, + "support.apple.com": 7, + "blogs.gentoo.org": 7, + "bitbucket.org": 7, + "bugzilla.suse.com": 7, + "go.dev": 7, + "pivotal.io": 7, + "whitesourcesoftware.com": 7, + "cdn.kernel.org": 7, + "gitee.com": 7, + "lore.kernel.org": 7, + "ruby-lang.org": 7, + "msrc.microsoft.com": 7, + "wordpress.org": 7, + "plone.org": 7, + "redhat.com": 7, + "oracle.com": 7, + "anonsvn.wireshark.org": 7, + "mail-archives.apache.org": 7, + "xenbits.xen.org": 6, + "postgresql.org": 6, + "twitter.com": 6, + "git.openssl.org": 6, + "jvn.jp": 6, + "rubygems.org": 6, + "symfony.com": 6, + "www-01.ibm.com": 6, + "src.chromium.org": 6, + "tenable.com": 6, + "lists.apple.com": 6, + "packagist.org": 6, + "issues.redhat.com": 6, + "helpx.adobe.com": 6, + "phabricator.wikimedia.org": 6, + "review.openstack.org": 6, + "chromereleases.googleblog.com": 6, + "contao.org": 6, + "discuss.elastic.co": 6, + "security.openstack.org": 6, + "lkml.org": 6, + "medium.com": 6, + "nozominetworks.com": 6, + "grafana.com": 6, + "oval.cisecurity.org": 6, + "review.opendev.org": 6, + "advisory.checkmarx.net": 6, + "vupen.com": 6, + "launchpad.net": 6, + "openssl.org": 6, + "framework.zend.com": 6, + "samba.org": 6, + "talosintelligence.com": 6, + "zerodayinitiative.com": 6, + "bugs.eclipse.org": 6, + "gitlab.gnome.org": 6, + "nifi.apache.org": 6, + "x-stream.github.io": 6, + "tanzu.vmware.com": 6, + "jvndb.jvn.jp": 6, + "opendev.org": 6, + "bugzilla.kernel.org": 6, + "sourceware.org": 6, + "codereview.chromium.org": 6, + "research.jfrog.com": 6, + "discuss.rubyonrails.org": 6, + "spring.io": 6, + "twcert.org.tw": 6, + "googlechromereleases.blogspot.com": 6, + "sec.cloudapps.cisco.com": 6, + "issues.jboss.org": 6, + "simplesamlphp.org": 6, + "mail-archive.com": 6, + "docs.saltstack.com": 6, + "docs.shopware.com": 6, + "kb.isc.org": 6, + "support.f5.com": 6, + "kernel.org": 6, + "osvdb.org": 6, + "cwiki.apache.org": 6, + "hg.moinmo.in": 6, + "fluidattacks.com": 6, + "gitlab.eclipse.org": 6, + "nuget.org": 6, + "lists.openstack.org": 6, + "developer.mozilla.org": 6, + "youtube.com": 6, + "drive.google.com": 6, + "owasp.org": 6, + "vaadin.com": 6, + "mercurial-scm.org": 6, + "mandriva.com": 6, + "archives.neohapsis.com": 6, + "hg.code.sf.net": 6, + "curl.haxx.se": 6, + "pillow.readthedocs.io": 6, + "struts.apache.org": 6, + "vulncheck.com": 6, + "webkitgtk.org": 6, + "documentation.concretecms.org": 6, + "patchstack.com": 6, + "patchwork.kernel.org": 6, + "hg.graphicsmagick.org": 6, + "source.android.com": 6, + "liferay.dev": 6, + "nodesecurity.io": 6, + "web.nvd.nist.gov": 6, + "jira.mongodb.org": 6, + "cve.org": 6, + "core.trac.wordpress.org": 6, + "news.ycombinator.com": 6, + "downloads.asterisk.org": 6, + "tomcat.apache.org": 6, + "kb.cert.org": 6, + "cert-portal.siemens.com": 6, + "devhub.checkmarx.com": 6, + "lists.nongnu.org": 6, + "squid-cache.org": 6, + "ink-desk-28f.notion.site": 6, + "h20566.www2.hpe.com": 6, + "intel.com": 6, + "docs.djangoproject.com": 6, + "go.googlesource.com": 6, + "herolab.usd.de": 6, + "matrix.org": 6, + "talosintel.com": 6, + "vapid.dhs.org": 6, + "ckeditor.com": 6, + "cwe.mitre.org": 6, + "pagure.io": 6, + "cisa.gov": 6, + "weblog.rubyonrails.org": 6, + "markmail.org": 6, + "issues.liferay.com": 6, + "jspwiki-wiki.apache.org": 6, + "gitbox.apache.org": 6, + "bugs.gentoo.org": 5, + "docs.microsoft.com": 5, + "puppet.com": 5, + "incibe.es": 5, + "cxf.apache.org": 5, + "bugzilla.wikimedia.org": 5, + "mvnrepository.com": 5, + "usom.gov.tr": 5, + "bz.apache.org": 5, + "nodejs.org": 5, + "syzkaller.appspot.com": 5, + "vuln.ryotak.me": 5, + "docs.google.com": 5, + "spinics.net": 5, + "auth0.com": 5, + "basercms.net": 5, + "next-auth.js.org": 5, + "plugins.jenkins.io": 5, + "libreoffice.org": 5, + "tiny.cloud": 5, + "hg.openjdk.java.net": 5, + "owncloud.org": 5, + "community.otrs.com": 5, + "git.openstack.org": 5, + "osgeo-org.atlassian.net": 5, + "bugzilla.gnome.org": 5, + "forum.xpdfreader.com": 5, + "istio.io": 5, + "developers.ibexa.co": 5, + "portswigger.net": 5, + "wpvulndb.com": 5, + "issues.chromium.org": 5, + "advisories.mageia.org": 5, + "doc.powerdns.com": 5, + "patchwork.ozlabs.org": 5, + "ec-cube.net": 5, + "phpbb.com": 5, + "rfc-editor.org": 5, + "synology.com": 5, + "ocert.org": 5, + "mail-archives.us.apache.org": 5, + "crates.io": 5, + "en.wikipedia.org": 5, + "issues.asterisk.org": 5, + "jira.spring.io": 5, + "secuniaresearch.flexerasoftware.com": 5, + "tools.ietf.org": 5, + "wiki.jenkins-ci.org": 5, + "youtu.be": 5, + "camel.apache.org": 5, + "0dd.zone": 5, + "blog.gitea.io": 5, + "bugs.ghostscript.com": 5, + "bugzilla.novell.com": 5, + "developer.joomla.org": 5, + "gitlab.torproject.org": 5, + "lists.wikimedia.org": 5, + "trac.torproject.org": 5, + "wpscan.com": 5, + "ibm.com": 5, + "trac.roundcube.net": 5, + "pidgin.im": 5, + "git.ghostscript.com": 5, + "gerrit.wikimedia.org": 5, + "gitlab.freedesktop.org": 5, + "hiddenlayer.com": 5, + "support.hpe.com": 5, + "thread.gmane.org": 5, + "cvcn.gov.it": 5, + "activemq.apache.org": 5, + "article.gmane.org": 5, + "bakery.cakephp.org": 5, + "bugs.python.org": 5, + "chromium.googlesource.com": 5, + "docs.github.com": 5, + "docs.rs": 5, + "grimthereaperteam.medium.com": 5, + "mailman.nginx.org": 5, + "psirt.global.sonicwall.com": 5, + "selenic.com": 5, + "esecforte.com": 5, + "mend.io": 5, + "netsparker.com": 5, + "playframework.com": 5, + "securityreason.com": 5, + "files.opcfoundation.org": 5, + "lgtm.com": 5, + "stackoverflow.com": 5, + "tools.cisco.com": 5, + "redmine.org": 5, + "vicarius.io": 5, + "w1.fi": 5, + "blog.phusion.nl": 5, + "codeigniter4.github.io": 5, + "csirt.divd.nl": 5, + "documentation.centreon.com": 5, + "eprint.iacr.org": 5, + "git.gnome.org": 5, + "joel-malwarebenchmark.github.io": 5, + "magento.com": 5, + "mantisbt.org": 5, + "research.loginsoft.com": 5, + "neos.io": 5, + "syss.de": 5, + "xforce.iss.net": 5, + "advisories.nats.io": 5, + "bugs.tryton.org": 5, + "lists.launchpad.net": 5, + "salsa.debian.org": 5, + "saltproject.io": 5, + "shopware.com": 5, + "shawroot.cc": 5, + "blog.ripstech.com": 5, + "blog.sonarsource.com": 5, + "cheatsheetseries.owasp.org": 5, + "docs.craftercms.org": 5, + "docs.umbraco.com": 5, + "go-review.googlesource.com": 5, + "kc.mcafee.com": 5, + "labs.integrity.pt": 5, + "pyup.io": 5, + "rubysec.com": 5, + "support.ntp.org": 5, + "wiki.openstack.org": 5, + "concretecms.org": 5, + "darkmatter.ae": 5, + "linkedin.com": 5, + "otrs.com": 5, + "privoxy.org": 5, + "sonarsource.com": 5, + "trustwave.com": 5, + "vmware.com": 5, + "blog.rubygems.org": 5, + "karmainsecurity.com": 5, + "cloud.google.com": 5, + "docs.python.org": 5, + "git-wip-us.apache.org": 5, + "nvidia.custhelp.com": 5, + "pastebin.com": 5, + "pypi.python.org": 5, + "python-security.readthedocs.io": 5, + "rumbling-slice-eb0.notion.site": 5, + "rapid7.com": 5, + "advisory.dw1.io": 5, + "akka.io": 5, + "android.googlesource.com": 5, + "argo-cd.readthedocs.io": 5, + "blog.fuzzing-project.org": 5, + "blog.laravel.com": 5, + "codeberg.org": 5, + "docs.cilium.io": 5, + "docs.docker.com": 5, + "docs.wagtail.org": 5, + "downloads.powerdns.com": 5, + "electronjs.org": 5, + "forums.rancher.com": 5, + "irssi.org": 5, + "kallithea-scm.org": 5, + "legalhackers.com": 5, + "lists.freedesktop.org": 5, + "mellium.im": 5, + "rubyonrails.org": 5, + "spark.apache.org": 5, + "storyboard.openstack.org": 5, + "subversion.apache.org": 5, + "tls.mbed.org": 5, + "sunsolve.sun.com": 5, + "wiki.ubuntu.com": 5, + "apollographql.com": 5, + "herodevs.com": 5, + "imagemagick.org": 5, + "kde.org": 5, + "qualys.com": 5, + "usenix.org": 5, + "wizlynxgroup.com": 5, + "x41-dsec.de": 5, + "gentoo.org": 5, + "git.videolan.org": 5, + "moinmo.in": 5, + "puppetlabs.com": 5, + "arxiv.org": 5, + "blog.getbootstrap.com": 5, + "review.gluster.org": 5, + "roy.marples.name": 5, + "stackblitz.com": 5, + "facebook.com": 5, + "git.savannah.gnu.org": 5, + "blog.torproject.org": 5, + "codepen.io": 5, + "codeql.github.com": 5, + "commons.apache.org": 5, + "confluence.atlassian.com": 5, + "cs.opensource.google": 5, + "doc.akka.io": 5, + "docs.opennms.com": 5, + "docs.openstack.org": 5, + "docs.silverstripe.org": 5, + "doc.traefik.io": 5, + "doyensec.com": 5, + "foss.heptapod.net": 5, + "gcc.gnu.org": 5, + "git.sheetjs.com": 5, + "hackmd.io": 5, + "ics-cert.kaspersky.com": 5, + "kiwitcms.org": 5, + "kjur.github.io": 5, + "lists.gnupg.org": 5, + "logback.qos.ch": 5, + "logging.apache.org": 5, + "mail.python.org": 5, + "metacpan.org": 5, + "neo4j.com": 5, + "opcfoundation.org": 5, + "ostorlab.co": 5, + "palletsprojects.com": 5, + "portal.liferay.dev": 5, + "research.securitum.com": 5, + "sites.google.com": 5, + "source.codeaurora.org": 5, + "support.sonatype.com": 5, + "twistedmatrix.com": 5, + "edoardoottavianelli.it": 5, + "sec-consult.com": 5, + "sudo.ws": 5, + "vulnerability-lab.com": 5, + "yiiframework.com": 5, + "www-1.ibm.com": 5, + "cherrypy.org": 5, + "vapidlabs.com": 5, + "archiva.apache.org": 5, + "bugs.cacti.net": 5, + "docs.moodle.org": 5, + "freeradius.org": 5, + "git.php.net": 5, + "hmarco.org": 5, + "hyp3rlinx.altervista.org": 5, + "lists.bestpractical.com": 5, + "php.net": 5, + "qpid.apache.org": 5, + "blog.clamav.net": 5, + "codex.wordpress.org": 5, + "cxsecurity.com": 5, + "security.libvirt.org": 5, + "extensions.typo3.org": 5, + "gnunet.org": 5, + "pretix.eu": 5, + "software.intel.com": 5, + "syncope.apache.org": 5, + "apache.org": 5, + "blackhat.com": 5, + "bleepingcomputer.com": 5, + "cyrusimap.org": 5, + "openssh.com": 5, + "arubanetworks.com": 5, + "code610.blogspot.com": 5, + "code.djangoproject.com": 5, + "karaf.apache.org": 5, + "lists.openwall.net": 5, + "netty.io": 5, + "packetstormsecurity.org": 5, + "arstechnica.com": 5, + "aws.amazon.com": 5, + "benjamin-bouchet.com": 5, + "blog.jquery.com": 5, + "bugs.jquery.com": 5, + "bugs.php.net": 5, + "bugs.torproject.org": 5, + "cert.pl": 5, + "community.grafana.com": 5, + "community.shopware.com": 5, + "cves.at": 5, + "datatracker.ietf.org": 5, + "docs.geoserver.org": 5, + "docs.sentry.io": 5, + "edugit.org": 5, + "fedorahosted.org": 5, + "fetch.spec.whatwg.org": 5, + "froala.com": 5, + "gerrit.libreoffice.org": 5, + "git1-us-west.apache.org": 5, + "grimhacker.com": 5, + "hg.reportlab.com": 5, + "kb.pulsesecure.net": 5, + "laravel.com": 5, + "lwn.net": 5, + "mail.openvswitch.org": 5, + "matrix-org.github.io": 5, + "nukeviet.vn": 5, + "portal.microfocus.com": 5, + "research.nccgroup.com": 5, + "rt.cpan.org": 5, + "s.apache.org": 5, + "search.maven.org": 5, + "security.docs.wso2.com": 5, + "share.ez.no": 5, + "shibboleth.net": 5, + "solr.apache.org": 5, + "starlabs.sg": 5, + "support.microsoft.com": 5, + "support.zabbix.com": 5, + "sysdream.com": 5, + "tech.feedyourhead.at": 5, + "dolibarr.org": 5, + "hashicorp.com": 5, + "opennms.com": 5, + "passbolt.com": 5, + "reddit.com": 5, + "sqlite.org": 5, + "strongswan.org": 5, + "hawktesters.com": 5, + "topquadrant.com": 5, + "blog.mindedsecurity.com": 5, + "dev.eclipse.org": 5, + "downloads.digium.com": 5, + "permalink.gmane.org": 5, + "0xacab.org": 5, + "about.gitlab.com": 5, + "alephsecurity.com": 5, + "blog.cloudflare.com": 5, + "bugs.freedesktop.org": 5, + "bugs.kde.org": 5, + "community.openvpn.net": 5, + "core.spip.net": 5, + "docs.oracle.com": 5, + "dumpco.re": 5, + "edk2-docs.gitbooks.io": 5, + "forum.cosmos.network": 5, + "git.ffmpeg.org": 5, + "golang.org": 5, + "hex.pm": 5, + "lists.x.org": 5, + "me.sap.com": 5, + "pdfium.googlesource.com": 5, + "peps.python.org": 5, + "portal.perforce.com": 5, + "roundcube.net": 5, + "trac.ffmpeg.org": 5, + "trac.pjsip.org": 5, + "svn.cacti.net": 5, + "couchbase.com": 5, + "nlnetlabs.nl": 5, + "prevanders.net": 5, + "sourceclear.com": 5, + "suse.com": 5, + "novell.com": 5, + "openafs.org": 5, + "tryton.org": 5, + "zope.org": 5, + "blog.plataformatec.com.br": 5, + "blog.trendmicro.com": 5, + "geronimo.apache.org": 5, + "hg.tryton.org": 5, + "issues.roundup-tracker.org": 5, + "akerva.com": 5, + "api.slack.com": 5, + "backdropcms.org": 5, + "blog.certimetergroup.com": 5, + "blog.jetbrains.com": 5, + "codereview.qt-project.org": 5, + "codewhitesec.blogspot.com": 5, + "community.librenms.org": 5, + "craftcms.com": 5, + "discord.com": 5, + "discuss.flarum.org": 5, + "discuss.tryton.org": 5, + "dl.acm.org": 5, + "docs.aws.amazon.com": 5, + "docs.pinot.apache.org": 5, + "docs.projectdiscovery.io": 5, + "documentation.bonitasoft.com": 5, + "duo.com": 5, + "elixirforum.com": 5, + "extensions.xwiki.org": 5, + "ezplatform.com": 5, + "git.launchpad.net": 5, + "gitlord.com": 5, + "git.shibboleth.net": 5, + "gnutls.org": 5, + "gxx777.github.io": 5, + "ibb.co": 5, + "i.ibb.co": 5, + "internetcomputer.org": 5, + "issues.opennms.org": 5, + "jira.qos.ch": 5, + "jsoup.org": 5, + "kiali.io": 5, + "lab.louiz.org": 5, + "labs.bishopfox.com": 5, + "linkis.apache.org": 5, + "mattermost.com": 5, + "mycvee.blogspot.com": 5, + "notes.sjtu.edu.cn": 5, + "opcfoundation-onlineapplications.org": 5, + "openid.net": 5, + "patch-diff.githubusercontent.com": 5, + "pho3n1x-web.github.io": 5, + "podalirius.net": 5, + "portals.apache.org": 5, + "projects.eclipse.org": 5, + "pulsesecurity.co.nz": 5, + "quarkus.io": 5, + "reactjs.org": 5, + "review.typo3.org": 5, + "salvatoresecurity.com": 5, + "securelayer7.net": 5, + "secure.ucc.asn.au": 5, + "shrouded-trowel-50c.notion.site": 5, + "ssd-disclosure.com": 5, + "strapi.io": 5, + "surrealdb.com": 5, + "tailscale.com": 5, + "tufangungor.github.io": 5, + "support.avaya.com": 5, + "webargs.readthedocs.io": 5, + "whitehub.net": 5, + "ait.ac.at": 5, + "checkmarx.com": 5, + "cloudbees.com": 5, + "cloudera.com": 5, + "codeigniter.com": 5, + "digital.security": 5, + "dulwich.io": 5, + "freeipa.org": 5, + "gerritcodereview.com": 5, + "ghostccamm.com": 5, + "horizon3.ai": 5, + "imperva.com": 5, + "mediawiki.org": 5, + "openmicroscopy.org": 5, + "schedmd.com": 5, + "wwws.nightwatchcybersecurity.com": 5, + "synacktiv.com": 5, + "synopsys.com": 5, + "tigera.io": 5, + "twistlock.com": 5, + "w3.org": 5, + "verneet.com": 5, + "dnnsoftware.com": 5, + "halfdog.net": 5, + "hitachi.co.jp": 5, + "reviewboard.org": 5, + "us-cert.gov": 5, + "depot.galaxyproject.org": 5, + "codecanyon.net": 5, + "marketplace.visualstudio.com": 5, + "docs.telerik.com": 5, + "support.hcltechsw.com": 5, + "watchguard.com": 5, + "bazaar.launchpad.net": 5, + "comments.gmane.org": 5, + "db.apache.org": 5, + "debbugs.gnu.org": 5, + "emberjs.com": 5, + "git.ganeti.org": 5, + "lists.thekelleys.org.uk": 5, + "phpmyadmin.git.sourceforge.net": 5, + "projects.puppetlabs.com": 5, + "anonscm.debian.org": 5, + "apereo.github.io": 5, + "blogs.apache.org": 5, + "bugzilla.opensuse.org": 5, + "build.prestashop.com": 5, + "community.rapid7.com": 5, + "cryptography.io": 5, + "cybersecurityworks.com": 5, + "dev.mysql.com": 5, + "docs.opencast.org": 5, + "docs.pagure.org": 5, + "docs.powerdns.com": 5, + "ftp.openbsd.org": 5, + "gitlab.labs.nic.cz": 5, + "git.linuxtv.org": 5, + "git.spip.net": 5, + "googleprojectzero.blogspot.com": 5, + "helm.sh": 5, + "hermes.opensuse.org": 5, + "hg.prosody.im": 5, + "isc.sans.edu": 5, + "issues.jenkins-ci.org": 5, + "jfrog.com": 5, + "jira.hdfgroup.org": 5, + "jira.springsource.org": 5, + "lists.01.org": 5, + "lists.lysator.liu.se": 5, + "lists.torproject.org": 5, + "nealpoole.com": 5, + "nokogiri.org": 5, + "novysodope.github.io": 5, + "somevulnsofadlab.blogspot.jp": 5, + "source.ikiwiki.branchable.com": 5, + "patchwork.linuxtv.org": 5, + "prosody.im": 5, + "pub.freerdp.com": 5, + "research.checkpoint.com": 5, + "rt-solutions.de": 5, + "tracker.debian.org": 5, + "user-images.githubusercontent.com": 5, + "ambionics.io": 5, + "bouncycastle.org": 5, + "freebsd.org": 5, + "mautic.org": 5, + "theregister.com": 5, + "trac.edgewall.org": 5, + "fujitsu.com": 5, + "gopivotal.com": 5, + "huawei.com": 5, + "ush.it": 5, + "wiki.shikangsi.com": 5, + "xenbits.xenproject.org": 5, + "bytium.com": 5, + "enpass.io": 5, + "archives.seul.org": 5, + "blog.npmjs.org": 5, + "blog.talosintel.com": 5, + "bugs.jqueryui.com": 5, + "dev.rubyonrails.org": 5, + "download.lighttpd.net": 5, + "freeimage.cvs.sourceforge.net": 5, + "googlechromereleases.blogspot.ca": 5, + "ijbswa.cvs.sourceforge.net": 5, + "issues.umbraco.org": 5, + "jira.codehaus.org": 5, + "jruby.org": 5, + "lcamtuf.coredump.cx": 5, + "lists.gnutls.org": 5, + "lists.qt-project.org": 5, + "lists.xen.org": 5, + "mx.gw.com": 5, + "octobercms.com": 5, + "openmeetings.apache.org": 5, + "pear.php.net": 5, + "projects.theforeman.org": 5, + "research.cs.wisc.edu": 5, + "airflow.apache.org": 5, + "santuario.apache.org": 5, + "appcheck-ng.com": 5, + "app.safebase.io": 5, + "app.snyk.io": 5, + "argoproj.github.io": 5, + "artifacthub.io": 5, + "authjs.dev": 5, + "savannah.nongnu.org": 5, + "bbpress.org": 5, + "belong2yourself.github.io": 5, + "blocksecteam.medium.com": 5, + "blog.cal1.cn": 5, + "blog.detectify.com": 5, + "blog.doyensec.com": 5, + "blog.gitea.com": 5, + "blog.jcoglan.com": 5, + "blog.jitendrapatro.me": 5, + "blog.jqueryui.com": 5, + "blog.jupyter.org": 5, + "blog.moku.fr": 5, + "blog.payara.fish": 5, + "blogs.securiteam.com": 5, + "blog.trailofbits.com": 5, + "breakingthe3ma.app": 5, + "brooklyn.apache.org": 5, + "buddypress.org": 5, + "bugs.oxid-esales.com": 5, + "bz.mercurial-scm.org": 5, + "cgit.kde.org": 5, + "channels.readthedocs.io": 5, + "chromium-review.googlesource.com": 5, + "ckan.org": 5, + "clojure.atlassian.net": 5, + "clusterfuzz-external.appspot.com": 5, + "community.snowflake.com": 5, + "community.sonarsource.com": 5, + "community.synopsys.com": 5, + "cupc4k3.medium.com": 5, + "developer.arm.com": 5, + "dev.liferay.com": 5, + "discuss.lightbend.com": 5, + "docs.getdbt.com": 5, + "docs.gradle.com": 5, + "docs.jboss.org": 5, + "docs.magnolia-cms.com": 5, + "docs.mattermost.com": 5, + "docs.opendaylight.org": 5, + "docs.rundeck.com": 5, + "docs.sixlabors.com": 5, + "docs.strapi.io": 5, + "docs.typo3.org": 5, + "docs.wso2.com": 5, + "duartecsantos.github.io": 5, + "eclipse.dev": 5, + "eslint.org": 5, + "excellium-services.com": 5, + "exment.net": 5, + "fatihhcelik.blogspot.com": 5, + "fortiguard.com": 5, + "forum.ghost.org": 5, + "forum.mautic.org": 5, + "gerrit.onosproject.org": 5, + "geth.ethereum.org": 5, + "ghost.org": 5, + "gist.githubusercontent.com": 5, + "git.eclipse.org": 5, + "git.haproxy.org": 5, + "github.dev": 5, + "github.openssl.org": 5, + "gitlab.kitware.com": 5, + "gitlab.ow2.org": 5, + "gitlab.redox-os.org": 5, + "git.opendaylight.org": 5, + "git-scm.com": 5, + "gitweb.gentoo.org": 5, + "gitweb.torproject.org": 5, + "glassfish.org": 5, + "go-vela.github.io": 5, + "grails.org": 5, + "gstreamer.freedesktop.org": 5, + "guidovranken.wordpress.com": 5, + "hadoop.apache.org": 5, + "hexdocs.pm": 5, + "hg.python.org": 5, + "htcondor-wiki.cs.wisc.edu": 5, + "ihacktoprotect.com": 5, + "ikiwiki.info": 5, + "infosecwriteups.com": 5, + "inspector.pypi.io": 5, + "invdos.net": 5, + "iotaa.cn": 5, + "ipsilon-project.org": 5, + "issues.shibboleth.net": 5, + "james.apache.org": 5, + "javaee.github.io": 5, + "jdbc.postgresql.org": 5, + "jekyllrb.com": 5, + "jena.apache.org": 5, + "jira.duraspace.org": 5, + "jira.hyperledger.org": 5, + "johnjhacking.com": 5, + "jolokia.org": 5, + "jupyterhub.readthedocs.io": 5, + "knightlab.northwestern.edu": 5, + "know.bishopfox.com": 5, + "kramdown.gettalong.org": 5, + "kubernetes.io": 5, + "kyverno.io": 5, + "laravel-news.com": 5, + "launchpadlibrarian.net": 5, + "learn.microsoft.com": 5, + "lf-opendaylight.atlassian.net": 5, + "lists.exim.org": 5, + "lists.quagga.net": 5, + "lists.samba.org": 5, + "lists.schedmd.com": 5, + "loopback.io": 5, + "mail.zope.org": 5, + "makandracards.com": 5, + "mariadb.com": 5, + "mosquitto.org": 5, + "mp.weixin.qq.com": 5, + "openfga.dev": 5, + "oxidforge.org": 5, + "pentest.com.tr": 5, + "phab.bots.miraheze.wiki": 5, + "plugins.craftcms.com": 5, + "polarssl.org": 5, + "spreecommerce.com": 5, + "public-inbox.org": 5, + "rancher.com": 5, + "redmine.openinfosecfoundation.org": 5, + "repo.mercurial-scm.org": 5, + "rhinosecuritylabs.com": 5, + "runkit.com": 5, + "sansec.io": 5, + "savannah.gnu.org": 5, + "seanmonstar.com": 5, + "security.FreeBSD.org": 5, + "seemann.io": 5, + "sl1nki.page": 5, + "sources.debian.org": 5, + "stackdiary.com": 5, + "support.hazelcast.com": 5, + "supportportal.juniper.net": 5, + "support.servicenow.com": 5, + "swarm.ptsecurity.com": 5, + "tauri.app": 5, + "thecybergeek.co.uk": 5, + "tib36.github.io": 5, + "storm.apache.org": 5, + "trovent.github.io": 5, + "trovent.io": 5, + "uima.apache.org": 5, + "unit42.paloaltonetworks.com": 5, + "unparalleled.eu": 5, + "support.blackberry.com": 5, + "uwsgi-docs.readthedocs.io": 5, + "vufind.org": 5, + "vulmon.com": 5, + "vuln.go.dev": 5, + "vuln.ryotak.net": 5, + "websecnerd.blogspot.in": 5, + "wha13.github.io": 5, + "whitehatck01.blogspot.com": 5, + "wiki.mageia.org": 5, + "wiki.phpbb.com": 5, + "wp-crontrol.com": 5, + "ag-grid.com": 5, + "apiman.io": 5, + "bentley.com": 5, + "bitvise.com": 5, + "bluetooth.com": 5, + "bookstackapp.com": 5, + "cnblogs.com": 5, + "cnvd.org.cn": 5, + "code-intelligence.com": 5, + "compass-security.com": 5, + "django-cms.org": 5, + "doctrine-project.org": 5, + "eclipse.org": 5, + "elastic.co": 5, + "envoyproxy.io": 5, + "exploitalert.com": 5, + "fastly.com": 5, + "fzi.de": 5, + "getastra.com": 5, + "graylog.org": 5, + "htbridge.com": 5, + "kingkk.com": 5, + "libssh.org": 5, + "nccgroup.trust": 5, + "nginx.com": 5, + "openpolicyagent.org": 5, + "pac4j.org": 5, + "pgadmin.org": 5, + "phpmyfaq.de": 5, + "roundup-tracker.org": 5, + "rubydoc.info": 5, + "saltstack.com": 5, + "secpod.com": 5, + "shielder.it": 5, + "smarty.net": 5, + "smilecdr.com": 5, + "soluble.ai": 5, + "star123.top": 5, + "sumor.top": 5, + "swascan.com": 5, + "teeworlds.com": 5, + "verot.net": 5, + "videolan.org": 5, + "xwiki.org": 5, + "yuque.com": 5, + "yarnpkg.com": 5, + "zeppelin.apache.org": 5, + "unomi.apache.org": 5, + "web2py.com": 5, + "web.mit.edu": 5, + "chiark.greenend.org.uk": 5, + "conostix.com": 5, + "daimacn.com": 5, + "dojotoolkit.org": 5, + "igniterealtime.org": 5, + "logilab.org": 5, + "swi-prolog.org": 5, + "yuiblog.com": 5, + "xerces.apache.org": 5, + "yuilibrary.com": 5, + "zaranshaikh.blogspot.com": 5, + "checkmk.com": 5, + "support.eset.com": 5, + "git.canopsis.net": 5, + "forescout.com": 5, + "mitel.com": 5, + "cert-in.org.in": 5, + "schneider-elektronik.de": 5, + "security-advisory.acronis.com": 5, + "activemq.2283324.n4.nabble.com": 5, + "apache-ignite-developers.2346864.n4.nabble.com": 5, + "baraktawily.blogspot.co.il": 5, + "blog.diniscruz.com": 5, + "blog.kotowicz.net": 5, + "blog.neargle.com": 5, + "blog.nodejs.org": 5, + "blog.talosintelligence.com": 5, + "borgbackup.readthedocs.io": 5, + "bugs.proftpd.org": 5, + "cgit.drupalcode.org": 5, + "erlang.org": 5, + "extplorer.net": 5, + "foxglovesecurity.com": 5, + "freshmeat.net": 5, + "googlechromereleases.blogspot.de": 5, + "h20000.www2.hp.com": 5, + "htmlpurifier.org": 5, + "itrc.hp.com": 5, + "kalilinux.co": 5, + "lists.horde.org": 5, + "lists.suse.com": 5, + "lists.ucc.gu.uwa.edu.au": 5, + "lkml.iu.edu": 5, + "mercurial.selenic.com": 5, + "miki.it": 5, + "openocd.zylin.com": 5, + "phpmyadmin.svn.sourceforge.net": 5, + "poi.apache.org": 5, + "redmine.lighttpd.net": 5, + "repo.or.cz": 5, + "rubyforge.org": 5, + "aka.ms": 5, + "sakurity.com": 5, + "blog.filippo.io": 5, + "blog.ircmaxell.com": 5, + "blog.justinbull.ca": 5, + "blogs.akamai.com": 5, + "blog.sonatype.com": 5, + "blog.spip.net": 5, + "blog.srcclr.com": 5, + "brakemanscanner.org": 5, + "bridge.grumpy-troll.org": 5, + "buer.haus": 5, + "bugs.dojotoolkit.org": 5, + "bugs.openjdk.java.net": 5, + "bugs.otr.im": 5, + "bugzilla.proxmox.com": 5, + "collectiveidea.com": 5, + "connect2id.com": 5, + "coredns.io": 5, + "cosmosofcyberspace.github.io": 5, + "delvingbitcoin.org": 5, + "devco.re": 5, + "dev.icinga.org": 5, + "docs.cometbft.com": 5, + "docs.couchbase.com": 5, + "docs.getindico.io": 5, + "dovecot.org": 5, + "download.samba.org": 5, + "forge.glpi-project.org": 5, + "ftp.suse.com": 5, + "git.blender.org": 5, + "git.enlightenment.org": 5, + "git.musl-libc.org": 5, + "git.samba.org": 5, + "git.zx2c4.com": 5, + "grsecurity.net": 5, + "hackdefense.com": 5, + "hg.mozilla.org": 5, + "hibernate.atlassian.net": 5, + "ics-cert.us-cert.gov": 5, + "int21.de": 5, + "issues.igniterealtime.org": 5, + "issues.rpath.com": 5, + "kb.juniper.net": 5, + "kyberslash.cr.yp.to": 5, + "labs.twistedmatrix.com": 5, + "ledgersmb.org": 5, + "lib.openmpt.org": 5, + "lists.dns-oarc.net": 5, + "lists.fedorahosted.org": 5, + "lists.linuxfoundation.org": 5, + "lists.xapian.org": 5, + "minerva.crocs.fi.muni.cz": 5, + "mta.openssl.org": 5, + "somevulnsofadlab.blogspot.com": 5, + "oss-fuzz.com": 5, + "oss.oracle.com": 5, + "source.git-annex.branchable.com": 5, + "pub.dev": 5, + "quickview.cloudapps.cisco.com": 5, + "robotattack.org": 5, + "security.googleblog.com": 5, + "shells.systems": 5, + "sintonen.fi": 5, + "skia.googlesource.com": 5, + "staaldraad.github.io": 5, + "store.shopware.com": 5, + "sumofpwn.nl": 5, + "support.lenovo.com": 5, + "trac.osgeo.org": 5, + "trac.webkit.org": 5, + "trac.xiph.org": 5, + "udiniya.wordpress.com": 5, + "support.novell.com": 5, + "weechat.org": 5, + "withatwist.dev": 5, + "codeaurora.org": 5, + "exim.org": 5, + "foxmole.com": 5, + "genivia.com": 5, + "gosecure.net": 5, + "home-assistant.io": 5, + "quagga.net": 5, + "tarlogic.com": 5, + "varnish-cache.org": 5, + "xorl.wordpress.com": 5, + "zenar.io": 5, + "zone.spip.net": 5, + "zyan.scripts.mit.edu": 5, + "theforeman.org": 5, + "tracker.firebirdsql.org": 5, + "trubka.network.cz": 5, + "wiki.mahara.org": 5, + "cacti.net": 5, + "eterna.com.au": 5, + "eweek.com": 5, + "ghostscript.com": 5, + "h5l.org": 5, + "ietf.org": 5, + "information-security.fr": 5, + "iwantacve.cn": 5, + "links.org": 5, + "mutt.org": 5, + "slackware.com": 5, + "thewildbeast.co.uk": 5, + "tornadoweb.org": 5, + "znuny.com": 5, + "apps.apple.com": 5, + "dontvacuum.me": 5, + "wiz.io": 5, + "tsc-soft.co.jp": 5, + "2016.hack.lu": 5, + "agrrrdog.blogspot.com": 5, + "alexcrack.com": 5, + "apache-spark-developers-list.1001551.n3.nabble.com": 5, + "appwrite.com": 5, + "archive.hack.lu": 5, + "argo.com": 5, + "bandoche.com": 5, + "bilishim.com": 5, + "blackboxexporter.com": 5, + "blog.amossys.fr": 5, + "blog.angularjs.org": 5, + "blog.apps.npr.org": 5, + "blog.bestpractical.com": 5, + "blog.checkpoint.com": 5, + "blog.csdn.net": 5, + "blog.datomic.com": 5, + "blog.emaze.net": 5, + "blog.intothesymmetry.com": 5, + "blog.portswigger.net": 5, + "blog.recurity-labs.com": 5, + "blog.securelayer7.net": 5, + "boltcms.com": 5, + "breaktoprotect.blogspot.com": 5, + "bugs.jython.org": 5, + "bugs.ledger-cli.org": 5, + "buildbot.net": 5, + "bzr.linuxfoundation.org": 5, + "ca17.com": 5, + "census-labs.com": 5, + "centreon.com": 5, + "ceriksen.com": 5, + "chargen.matasano.com": 5, + "chrony.tuxfamily.org": 5, + "code.call-cc.org": 5, + "community.ca.com": 5, + "community.impresscms.org": 5, + "cpansearch.perl.org": 5, + "craft.com": 5, + "crafter.com": 5, + "cvs.moodle.org": 5, + "cvsweb.netbsd.org": 5, + "cyberworldmirror.com": 5, + "davidsopaslabs.blogspot.com": 5, + "dev.deluge-torrent.org": 5, + "dfn.dl.sourceforge.net": 5, + "directory.apache.org": 5, + "doc.scrapy.org": 5, + "doc.silverstripe.org": 5, + "docs.info.apple.com": 5, + "docs.python-requests.org": 5, + "docs.withknown.com": 5, + "dolibarr.com": 5, + "dotclear.org": 5, + "dx.doi.org": 5, + "elixir.ematia.de": 5, + "embed.plnkr.co": 5, + "en.0day.today": 5, + "encode.com": 5, + "etcd.com": 5, + "ethereum.com": 5, + "evilpacket.net": 5, + "exfiltrated.com": 5, + "extendedsubset.com": 5, + "fitnesse.org": 5, + "flyingmana.de": 5, + "freecode.com": 5, + "ghost.com": 5, + "gilacms.com": 5, + "git-blame.blogspot.com": 5, + "git.deluge-torrent.org": 5, + "githubcommherflower.com": 5, + "git.postgresql.org": 5, + "go-ethereum.com": 5, + "gogs.io": 5, + "gollum.com": 5, + "grailsblog.objectcomputing.com": 5, + "groovy-lang.org": 5, + "ha.cker.info": 5, + "haxx.ml": 5, + "heartex.com": 5, + "help.plot.ly": 5, + "hidden-one.co.in": 5, + "holisticinfosec.org": 5, + "hsqldb.org": 5, + "icecoder.com": 5, + "info.tiki.org": 5, + "jakarta.apache.org": 5, + "jeecg-boot.com": 5, + "jinja.pocoo.org": 5, + "joplin.com": 5, + "juddi.apache.org": 5, + "labelstud.io": 5, + "labs.mwrinfosecurity.com": 5, + "lambdaops.com": 5, + "langchain.com": 5, + "lavalite.com": 5, + "lcamtuf.blogspot.ca": 5, + "lesscss.org": 5, + "libvirt.org": 5, + "liferay.com": 5, + "linux.oracle.com": 5, + "lists.live555.com": 5, + "lists.mutt.org": 5, + "lists.roaringpenguin.com": 5, + "lists.typo3.org": 5, + "lua.2524044.n2.nabble.com": 5, + "lxml.de": 5, + "maven.apache.org": 5, + "mina.apache.org": 5, + "mumble.info": 5, + "n8.tumblr.com": 5, + "nevado.skyscreamer.org": 5, + "ngenuity-is.com": 5, + "nginx.org": 5, + "nodejs.com": 5, + "openapi-generator.com": 5, + "openbsd.org": 5, + "opennlp.apache.org": 5, + "ph0rse.me": 5, + "processwire.com": 5, + "prometheus.com": 5, + "pyload.com": 5, + "pypi.doubanio.com": 5, + "pypinksign.com": 5, + "python.6.x6.nabble.com": 5, + "qcubed.com": 5, + "rails.lighthouseapp.com": 5, + "raneto.com": 5, + "request-baskets.com": 5, + "resque.com": 5, + "roundup.svn.sourceforge.net": 5, + "ruffsecurity.blogspot.com": 5, + "0ang3el.blogspot.ru": 5, + "2018.zeronights.ru": 5, + "98587329.github.io": 5, + "aaltodoc.aalto.fi": 5, + "accumulo.apache.org": 5, + "acloudtree.com": 5, + "adamcaudill.com": 5, + "advisories.gitlab.com": 5, + "advisories.octopus.com": 5, + "aetsu.github.io": 5, + "agent-js.icp.xyz": 5, + "sailsjs.org": 5, + "aisec.today": 5, + "aleksis.org": 5, + "alesandroortiz.com": 5, + "alexsecurity.rocks": 5, + "alicangonullu.org": 5, + "anisiosantos.me": 5, + "anongit.mindrot.org": 5, + "ant.apache.org": 5, + "antmedia.io": 5, + "apidoc.gitee.com": 5, + "apidock.com": 5, + "apiiro.com": 5, + "app.intigriti.com": 5, + "appwrite.io": 5, + "atlaskit.atlassian.com": 5, + "attackerkb.com": 5, + "at-trustit.tuv.at": 5, + "autobahn.readthedocs.io": 5, + "avideo.tube": 5, + "backbonejs.org": 5, + "bad.code.blog": 5, + "badpackets.net": 5, + "bandit.readthedocs.io": 5, + "baomidou.com": 5, + "beanvalidation.org": 5, + "becomepentester.blogspot.ae": 5, + "bishopfox.com": 5, + "bitbucket.hdfgroup.org": 5, + "bittherapy.net": 5, + "blakeembrey.com": 5, + "blitiri.com.ar": 5, + "blog.0xzon.dev": 5, + "blog.appsecco.com": 5, + "blog.bentkowski.info": 5, + "blog.bssi.fr": 5, + "blog.caller.xyz": 5, + "blog.chebuya.com": 5, + "blog.compass-security.com": 5, + "blog.daniel-ruf.de": 5, + "blog.effectrenan.com": 5, + "blog.ethereum.org": 5, + "blog.exodusintel.com": 5, + "blog.gdssecurity.com": 5, + "blog.gradle.org": 5, + "blog.hackeriet.no": 5, + "blog.hackingforce.com.br": 5, + "blog.ipfs.io": 5, + "blog.isosceles.com": 5, + "blog.jiguang.xyz": 5, + "blog.librenms.org": 5, + "blog.litespeedtech.com": 5, + "blog.mevsec.com": 5, + "blog.nettitude.com": 5, + "blog.orange.tw": 5, + "blog.ostorlab.co": 5, + "blog.phpbb.com": 5, + "blog.pusher.com": 5, + "blog.qualys.com": 5, + "blog.sentry.io": 5, + "blogs.gnome.org": 5, + "blog.slonser.info": 5, + "blog.smithsecurity.biz": 5, + "blogs.opera.com": 5, + "blogs.sap.com": 5, + "blog.truesec.com": 5, + "blog.xss.am": 5, + "boats.gitlab.io": 5, + "boho.or.kr": 5, + "book.hacktricks.xyz": 5, + "brave.com": 5, + "breakandpray.com": 5, + "bsg.tech": 5, + "bto.bluecoat.com": 5, + "bugs.bitlbee.org": 5, + "bugs.libssh.org": 5, + "bugs.limesurvey.org": 5, + "bugs.otrs.org": 5, + "bugs.ruby-lang.org": 5, + "bugzilla.libav.org": 5, + "bundler.io": 5, + "burninatorsec.blogspot.com": 5, + "buttercup.pw": 5, + "caddy.community": 5, + "caddyserver.com": 5, + "scala-lang.org": 5, + "cardaci.xyz": 5, + "carl1l.github.io": 5, + "scarybeastsecurity.blogspot.de": 5, + "casdoor.org": 5, + "c-c-a.org": 5, + "cdn2.hubspot.net": 5, + "cdn.datatables.net": 5, + "cdn.sheetjs.com": 5, + "cert.enea.pl": 5, + "cgit.freebsd.org": 5, + "cgit.freedesktop.org": 5, + "chaos.social": 5, + "chartkick.com": 5, + "checkstyle.org": 5, + "chmod744.super.site": 5, + "chocapikk.com": 5, + "clerk.com": 5, + "clojars.org": 5, + "cloud-trustit.spp.at": 5, + "codeburst.io": 5, + "code.jeremyevans.net": 5, + "codex.bbpress.org": 5, + "codex.buddypress.org": 5, + "com0t.github.io": 5, + "commonmark.thephpleague.com": 5, + "community.contao.org": 5, + "community.developer.atlassian.com": 5, + "community.gravitee.io": 5, + "community.traefik.io": 5, + "community.veracode.com": 5, + "concretecms.com": 5, + "contrastsecurity.com": 5, + "cortexmetrics.io": 5, + "cowtowncoder.medium.com": 5, + "cratedb.com": 5, + "cryptosense.com": 5, + "csrc.nist.gov": 5, + "cube01.io": 5, + "cupc4k3.lol": 5, + "cure53.de": 5, + "cve.anastasi.link": 5, + "cve.naver.com": 5, + "cve.nstsec.com": 5, + "cyllective.com": 5, + "danielfett.de": 5, + "dannewitz.ninja": 5, + "darkbit.io": 5, + "datasette.io": 5, + "datnlq.gitbook.io": 5, + "deadsh0t.medium.com": 5, + "demo.ripstech.com": 5, + "deps.dev": 5, + "devdocs.magento.com": 5, + "devel0pment.de": 5, + "developer.apple.com": 5, + "developer.hashicorp.com": 5, + "developer.jboss.org": 5, + "developers.cloudflare.com": 5, + "developers.google.com": 5, + "developer.shopware.com": 5, + "developers.yubico.com": 5, + "developer.woocommerce.com": 5, + "develop.sentry.dev": 5, + "dev.gajim.org": 5, + "dev.to": 5, + "diff.coditsu.io": 5, + "diff.hex.pm": 5, + "digi.ninja": 5, + "directus.io": 5, + "discourse.vtk.org": 5, + "discuss.istio.io": 5, + "dist.apache.org": 5, + "dist.plone.org": 5, + "dnsdist.org": 5, + "doc.clickup.com": 5, + "doc.rust-lang.org": 5, + "docs.centreon.com": 5, + "docs.ceph.com": 5, + "docs.ckan.org": 5, + "docs.dapr.io": 5, + "docs.dask.org": 5, + "docs.dependencytrack.org": 5, + "docs.directus.io": 5, + "docs.flyte.org": 5, + "docs.gitlab.com": 5, + "docs.gofiber.io": 5, + "docs.gravityforms.com": 5, + "docs.libp2p.io": 5, + "docs.locust.io": 5, + "docs.mulesoft.com": 5, + "docs.nautobot.com": 5, + "docs.nginx.com": 5, + "docs.parseplatform.org": 5, + "docs.pylonsproject.org": 5, + "docs.r3.com": 5, + "docs.saltproject.io": 5, + "docs.scrapy.org": 5, + "docs.securesauce.dev": 5, + "docs.sigstore.dev": 5, + "docs.snowflake.com": 5, + "docs.spring.io": 5, + "docs.sqlalchemy.org": 5, + "docs.sympy.org": 5, + "docs.tigergraph.com": 5, + "docs.totaljs.com": 5, + "docs.velociraptor.app": 5, + "docs.veracode.com": 5, + "docs.vyperlang.org": 5, + "docs.wagtail.io": 5, + "documentation.concrete5.org": 5, + "documentation.wazuh.com": 5, + "doi.org": 5, + "dotnetnuke.codeplex.com": 5, + "dubell.io": 5, + "dustri.org": 5, + "dwrensha.github.io": 5, + "security.360.cn": 5, + "security.openttd.org": 5, + "securitywarrior9.blogspot.com": 5, + "edg.io": 5, + "edhunter484.medium.com": 5, + "eldstal.se": 5, + "electron.atom.io": 5, + "elgg.org": 5, + "emily.id.au": 5, + "en.bandisoft.com": 5, + "en.bitcoin.it": 5, + "engindemirbilek.github.io": 5, + "en.osdn.jp": 5, + "epadillas.github.io": 5, + "erlef.github.io": 5, + "eslam.io": 5, + "exceptionfactory.com": 5, + "experienceleague.adobe.com": 5, + "expressjs.com": 5, + "fastapi-admin-pro.long2ice.io": 5, + "fbdhhhh47.github.io": 5, + "febin0x4e4a.blogspot.com": 5, + "febin0x4e4a.wordpress.com": 5, + "febinj.medium.com": 5, + "ferrous-systems.com": 5, + "fgsec.net": 5, + "filezilla-project.org": 5, + "fisheye6.atlassian.com": 5, + "flask-limiter.readthedocs.io": 5, + "floqast.com": 5, + "flowiseai.com": 5, + "flyd.uk": 5, + "fmyyy1.github.io": 5, + "forge.centreon.com": 5, + "forge.typo3.org": 5, + "forum.codeigniter.com": 5, + "forum.datomic.com": 5, + "forum.ksec.co.uk": 5, + "forum.netgate.com": 5, + "forum.openzeppelin.com": 5, + "forums.couchbase.com": 5, + "forum.silverstripe.org": 5, + "forums.swift.org": 5, + "fossies.org": 5, + "freemarker.apache.org": 5, + "frycos.github.io": 5, + "fuo.fi": 5, + "fusionauth.io": 5, + "gainsec.com": 5, + "gauravnarwani.com": 5, + "genix.me": 5, + "geoserver.org": 5, + "gerrit.asterisk.org": 5, + "gerrit.ovirt.org": 5, + "getcomposer.org": 5, + "getgrav.org": 5, + "getkirby.com": 5, + "getlaminas.org": 5, + "getlektor.com": 5, + "git-annex.branchable.com": 5, + "git.fedorahosted.org": 5, + "github.com.mattermost": 5, + "git.libssh.org": 5, + "git.oschina.net": 5, + "git.reviewboard.kde.org": 5, + "git.sdaoden.eu": 5, + "git.xiph.org": 5, + "gld.mcphail.uk": 5, + "glitch.com": 5, + "gluu.org": 5, + "go2docs.graylog.org": 5, + "godoc.org": 5, + "gohugo.io": 5, + "go-review.git.corp.google.com": 5, + "graz.pure.elsevier.com": 5, + "greysec.net": 5, + "groups.drupal.org": 5, + "grumpz.net": 5, + "guides.spreecommerce.org": 5, + "gultsch.de": 5, + "hackerdna.com": 5, + "hacker.soarescorp.com": 5, + "hackers.report": 5, + "hackinglab.cz": 5, + "hackmysystems.tumblr.com": 5, + "hackpuntes.com": 5, + "hansmi.ch": 5, + "hapifhir.io": 5, + "help.ecostruxureit.com": 5, + "help.egroupware.org": 5, + "help.mulesoft.com": 5, + "help.panic.com": 5, + "help.passbolt.com": 5, + "help.rapid7.com": 5, + "help.sonatype.com": 5, + "shindig.apache.org": 5, + "shiro.apache.org": 5, + "homakov.blogspot.ru": 5, + "html5lib.readthedocs.io": 5, + "httpwg.org": 5, + "hub.docker.com": 5, + "hugegraph.apache.org": 5, + "i.blackhat.com": 5, + "ibm.github.io": 5, + "igniterealtime.atlassian.net": 5, + "silverpeas.com": 5, + "imgur.com": 5, + "infosec.exchange": 5, + "infra.spec.whatwg.org": 5, + "inhann.top": 5, + "inlong.apache.org": 5, + "insinuator.net": 5, + "intrix.com.au": 5, + "ipython.org": 5, + "ipython.readthedocs.io": 5, + "irrd.readthedocs.io": 5, + "isec.pl": 5, + "issues.ibexa.co": 5, + "issues.jasig.org": 5, + "issues.jenkins.io": 5, + "issues.sonatype.org": 5, + "it-sec.de": 5, + "itsmeanonartist.tech": 5, + "jadaptive.com": 5, + "janino-compiler.github.io": 5, + "java.net": 5, + "jay-from-future.github.io": 5, + "jenkins-ci.org": 5, + "jetpack.com": 5, + "jinja.palletsprojects.com": 5, + "jinmu1108.github.io": 5, + "jira.onosproject.org": 5, + "jira.opendaylight.org": 5, + "jira.sonarsource.com": 5, + "jira.whamcloud.com": 5, + "jquery.com": 5, + "jqueryui.com": 5, + "jqueryvalidation.org": 5, + "jsfiddle.net": 5, + "jub0bs.com": 5, + "junit.org": 5, + "kafka.apache.org": 5, + "kb.hitcon.org": 5, + "kb.netapp.com": 5, + "skipper.com": 5, + "koji.fedoraproject.org": 5, + "korelogic.com": 5, + "koz.io": 5, + "labanskoller.se": 5, + "labs.detectify.com": 5, + "lab.wallarm.com": 5, + "landave.io": 5, + "laravel-admin.org": 5, + "latestpcsolution.wordpress.com": 5, + "launchpad.support.sap.com": 5, + "laworigin.github.io": 5, + "layui.dev": 5, + "learn.snyk.io": 5, + "lednerb.de": 5, + "lessonsec.com": 5, + "libgit2.org": 5, + "libnmap.readthedocs.io": 5, + "lib.rs": 5, + "lightning.network": 5, + "limpidsecurity.pl": 5, + "linkerd.io": 5, + "linotp.org": 5, + "lists.cncf.io": 5, + "lists.denx.de": 5, + "lists.ffmpeg.org": 5, + "lists.mindrot.org": 5, + "lists.osgeo.org": 5, + "lists.w3.org": 5, + "lists.zx2c4.com": 5, + "locutus.io": 5, + "lutrasecurity.com": 5, + "lycshub.github.io": 5, + "m3n0sd0n4ld.github.io": 5, + "mail.gnome.org": 5, + "mailman-mail5.webfaction.com": 5, + "smalruby.jp": 5, + "marketplace.atlassian.com": 5, + "martinthomson.github.io": 5, + "mat4mee.notion.site": 5, + "matt.ucc.asn.au": 5, + "mayaseven.com": 5, + "media.dedaub.com": 5, + "mensfeld.pl": 5, + "meshery.io": 5, + "meta.wikimedia.org": 5, + "meterpreter.org": 5, + "minhnq22.medium.com": 5, + "mitmproxy.org": 5, + "mitogen.networkgenomics.com": 5, + "mksec.tk": 5, + "mlflow.org": 5, + "modwsgi.readthedocs.io": 5, + "monicz.dev": 5, + "morehouse.github.io": 5, + "mostwanted002.cf": 5, + "mouha.be": 5, + "movermeyer.com": 5, + "msrc-blog.microsoft.com": 5, + "mthbernardes.github.io": 5, + "mulch.dev": 5, + "my.f5.com": 5, + "my.goanywhere.com": 5, + "nablarch.atlassian.net": 5, + "nakedsecurity.sophos.com": 5, + "nandynarwhals.org": 5, + "narrow-oatmeal-0c0.notion.site": 5, + "nasa.github.io": 5, + "nest.pijul.com": 5, + "nghttp2.org": 5, + "nmap.org": 5, + "nodemailer.com": 5, + "node-postgres.com": 5, + "nostarttls.secvuln.info": 5, + "notes.netbytesec.com": 5, + "note.youdao.com": 5, + "note.zhaoj.in": 5, + "nova.app": 5, + "nowotarski.info": 5, + "nozero.io": 5, + "nsfocusglobal.com": 5, + "numanozdemir.com": 5, + "objectcomputing.com": 5, + "octoprint.org": 5, + "okankurtulus.com.tr": 5, + "omespino.com": 5, + "onekey.com": 5, + "openai.com": 5, + "openbase.com": 5, + "opencast.jira.com": 5, + "opencirt.com": 5, + "openjdk.org": 5, + "opennms.atlassian.net": 5, + "opensearch.org": 5, + "opensource.fast-report.com": 5, + "openssf.org": 5, + "opentelemetry.io": 5, + "opsecx.com": 5, + "orc.apache.org": 5, + "oroinc.com": 5, + "oryx-embedded.com": 5, + "oss-security.openwall.narkive.com": 5, + "osv.dev": 5, + "owncloud.com": 5, + "packaging.python.org": 5, + "pagehelper.github.io": 5, + "panda002.hashnode.dev": 5, + "papers.mathyvanhoef.com": 5, + "sparkjava.com": 5, + "payatu.com": 5, + "peckshield.com": 5, + "pentesterlab.com": 5, + "people.kingsds.network": 5, + "petl.readthedocs.io": 5, + "piraeus.io": 5, + "plotly.com": 5, + "plugins.gradle.org": 5, + "plus.google.com": 5, + "pony7.fr": 5, + "popalltheshells.medium.com": 5, + "powerful-bulb-c36.notion.site": 5, + "prestosql.io": 5, + "pretalx.com": 5, + "prismjs.com": 5, + "privatebin.info": 5, + "programmer.help": 5, + "projects.duckcorp.org": 5, + "prophaze.com": 5, + "publicobject.com": 5, + "pubs.acs.org": 5, + "pugjs.org": 5, + "pulsar.apache.org": 5, + "pydio.com": 5, + "pythonhosted.org": 5, + "python-hyper.org": 5, + "quilljs.com": 5, + "quiltmc.org": 5, + "r0.haxors.org": 5, + "ranchermanager.docs.rancher.com": 5, + "rankmath.com": 5, + "rastating.github.io": 5, + "ratpack.io": 5, + "reportportal.io": 5, + "repo.saltproject.io": 5, + "researchgate.net": 5, + "research.hisolutions.com": 5, + "research.insecurelabs.org": 5, + "restsharp.dev": 5, + "review.whamcloud.com": 5, + "rhodecode.com": 5, + "rhynorater.github.io": 5, + "robertheaton.com": 5, + "rodelllemit.medium.com": 5, + "rootdaemon.com": 5, + "roumenpetrov.info": 5, + "rpyc.readthedocs.io": 5, + "ruby-doc.org": 5, + "rudnkh.me": 5, + "sahildhar.github.io": 5, + "samuzora.com": 5, + "saturncloud.io": 5, + "sca.analysiscenter.veracode.com": 5, + "scalyr-static.s3.amazonaws.com": 5, + "scikit-learn.org": 5, + "scnps.co": 5, + "scratch.mit.edu": 5, + "sec.stealthcopter.com": 5, + "secure1.securityspace.com": 5, + "security-garage.com": 5, + "security.gradle.com": 5, + "securitylabs.datadoghq.com": 5, + "security.paloaltonetworks.com": 5, + "securitypitfalls.wordpress.com": 5, + "securitytrails.com": 5, + "segment.com": 5, + "sektioneins.de": 5, + "sethmlarson.dev": 5, + "setuptools.pypa.io": 5, + "shattered.io": 5, + "sheetjs.com": 5, + "siebene.github.io": 5, + "silverpeas.org": 5, + "skalatan.de": 5, + "skerritt.blog": 5, + "skii.dev": 5, + "skywalking.apache.org": 5, + "sling.apache.org": 5, + "smshrimant.com": 5, + "snicco.io": 5, + "softwaresupport.hpe.com": 5, + "solidus.io": 5, + "sonarsource.atlassian.net": 5, + "sources.debian.net": 5, + "spacemesh.io": 5, + "specifications.freedesktop.org": 5, + "spec.matrix.org": 5, + "src.fedoraproject.org": 5, + "srcincite.io": 5, + "stackstorm.com": 5, + "staging-website.elastic.co": 5, + "stash.kopano.io": 5, + "subrion.org": 5, + "support.aerospike.com": 5, + "support.ca.com": 5, + "support.citrix.com": 5, + "support.confluent.io": 5, + "support.contrastsecurity.com": 5, + "support.delphix.com": 5, + "support.herodevs.com": 5, + "support.snyk.io": 5, + "sw.aveva.com": 5, + "sylabs.io": 5, + "sylius.com": 5, + "sympa-community.github.io": 5, + "sysdig.com": 5, + "systeminformation.io": 5, + "systemweakness.com": 5, + "tada.github.io": 5, + "tadayoshi-sato.medium.com": 5, + "tantosec.com": 5, + "tches.iacr.org": 5, + "techblog.wikimedia.org": 5, + "technet.microsoft.com": 5, + "tempered.works": 5, + "tf1t.gitbook.io": 5, + "thehackernews.com": 5, + "the-it-wonders.blogspot.com": 5, + "thelia.net": 5, + "threat.tevora.com": 5, + "threema.ch": 5, + "tickets.puppetlabs.com": 5, + "tiki.org": 5, + "stimulsoft.com": 5, + "tprynn.github.io": 5, + "tracker.die-offenbachs.homelinux.org": 5, + "tracker.moodle.org": 5, + "tracker.phpbb.com": 5, + "tracker.zkoss.org": 5, + "trac.xapian.org": 5, + "trafficcontrol.apache.org": 5, + "travis-ci.com": 5, + "trends.builtwith.com": 5, + "strimzi.com": 5, + "trino.io": 5, + "tristartom.github.io": 5, + "truedigitalsecurity.com": 5, + "trufflesecurity.com": 5, + "trungvm.gitbook.io": 5, + "trust.neo4j.com": 5, + "tryhexadecimal.com": 5, + "tttang.com": 5, + "unbound.net": 5, + "unpoly.com": 5, + "updates.playhive.com": 5, + "updates.snyk.io": 5, + "support.zeus.com": 5, + "suricata-ids.org": 5, + "us-cert.cisa.gov": 5, + "vapor.codes": 5, + "vega.github.io": 5, + "veracode.com": 5, + "verichains.io": 5, + "versprite.com": 5, + "vertx.io": 5, + "vimeo.com": 5, + "vitejs.dev": 5, + "svn.savannah.gnu.org": 5, + "vulners.com": 5, + "w3lib.readthedocs.io": 5, + "warehouse.python.org": 5, + "weakdh.org": 5, + "webassembly.github.io": 5, + "webiny.com": 5, + "webmasters.googleblog.com": 5, + "wger.de": 5, + "wicket.apache.org": 5, + "wiki.duraspace.org": 5, + "wiki.eclipse.org": 5, + "wiki.folio.org": 5, + "wiki.gentoo.org": 5, + "wiki.mercurial-scm.org": 5, + "wiki.opendaylight.org": 5, + "wiki.shibboleth.net": 5, + "winscp.net": 5, + "wintercms.com": 5, + "winterdragon.ca": 5, + "wiremock.org": 5, + "withknown.com": 5, + "wtfsec.org": 5, + "activecyber.us": 5, + "admidio.org": 5, + "adminer.org": 5, + "aleksey.com": 5, + "alevsk.com": 5, + "alluxio.io": 5, + "ansible.com": 5, + "anyscale.com": 5, + "apereo.org": 5, + "apolloconfig.com": 5, + "baeldung.com": 5, + "barracuda.com": 5, + "benthamsgaze.org": 5, + "bountysource.com": 5, + "brics.dk": 5, + "broadcom.com": 5, + "brzozowski.io": 5, + "bsi.bund.de": 5, + "ccsq8.com": 5, + "certik.com": 5, + "chtsecurity.com": 5, + "cigital.com": 5, + "ciphertechs.com": 5, + "cloudflare.com": 5, + "cloudfoundry.org": 5, + "coalfire.com": 5, + "concrete5.org": 5, + "coresecurity.com": 5, + "crowdstrike.com": 5, + "crushftp.com": 5, + "cvedetails.com": 5, + "cybereagle.io": 5, + "cyfirma.com": 5, + "darkreading.com": 5, + "ddosi.org": 5, + "descope.com": 5, + "dnspython.org": 5, + "docker.com": 5, + "doyler.net": 5, + "dsecbypass.com": 5, + "dubget.com": 5, + "economizzer.org": 5, + "edwardthomson.com": 5, + "elementary-data.com": 5, + "enhavo.com": 5, + "enterprisedb.com": 5, + "equalexperts.com": 5, + "eyecontrol.nl": 5, + "fastify.io": 5, + "ffmpeg.org": 5, + "fireeye.com": 5, + "fork-cms.com": 5, + "fossil-scm.org": 5, + "gitpod.io": 5, + "gradio.app": 5, + "gsma.com": 5, + "hacksecproject.com": 5, + "hackthebox.com": 5, + "hakaioffensivesecurity.com": 5, + "haproxy.com": 5, + "htmlunit.org": 5, + "ihteam.net": 5, + "imperialviolet.org": 5, + "impresscms.org": 5, + "intruder.io": 5, + "isc.org": 5, + "jetbrains.com": 5, + "jhipster.tech": 5, + "lancom-systems.de": 5, + "landaire.net": 5, + "laravel-enlightn.com": 5, + "limesurvey.org": 5, + "logicallysecure.com": 5, + "mageni.net": 5, + "makotemplates.org": 5, + "mandiant.com": 5, + "menlosecurity.com": 5, + "mgm-sp.com": 5, + "monolune.com": 5, + "mpg123.de": 5, + "nds.ruhr-uni-bochum.de": 5, + "netlify.com": 5, + "netsarang.com": 5, + "ni.com": 5, + "nomadproject.io": 5, + "nu11secur1ty.com": 5, + "obrela.com": 5, + "octobot.online": 5, + "okta.com": 5, + "on-x.com": 5, + "op-c.net": 5, + "opencart.com": 5, + "opencrx.org": 5, + "optiv.com": 5, + "oxeye.io": 5, + "papermerge.com": 5, + "paramiko.org": 5, + "pethuraj.com": 5, + "phoronix.com": 5, + "pizzapower.me": 5, + "portainer.io": 5, + "postfix.org": 5, + "www-prd-trops.events.ibm.com": 5, + "prestashop.com": 5, + "privacy-wise.com": 5, + "projectcalico.org": 5, + "purplemet.com": 5, + "qemu.org": 5, + "raifberkaydincel.com": 5, + "rejetto.com": 5, + "reportlab.com": 5, + "rsaconference.com": 5, + "rust-lang.org": 5, + "sap.com": 5, + "scottbrady91.com": 5, + "secureauth.com": 5, + "securitymetrics.com": 5, + "semantic-mediawiki.org": 5, + "shielder.com": 5, + "shorebreaksecurity.com": 5, + "sidertia.com": 5, + "slf4j.org": 5, + "sockjs.org": 5, + "splunk.com": 5, + "symantec.com": 5, + "sympa.org": 5, + "tcpdump.org": 5, + "telerik.com": 5, + "terrapin-attack.com": 5, + "trustmatta.com": 5, + "vandyke.com": 5, + "vdoo.com": 5, + "viralpatel.net": 5, + "virtualbox.org": 5, + "whitehats.nl": 5, + "winimage.com": 5, + "yubico.com": 5, + "zdnet.com": 5, + "zenml.io": 5, + "zofrex.com": 5, + "zsh.org": 5, + "xalan.apache.org": 5, + "xavibel.com": 5, + "xmit.xyz": 5, + "xmlgraphics.apache.org": 5, + "xmpp.org": 5, + "yondon.blog": 5, + "zerosecuritypenetrationtesting.com": 5, + "zh-cn.tenable.com": 5, + "zookeeper.apache.org": 5, + "zuso.ai": 5, + "zxsecurity.co.nz": 5, + "testh5shanglv.minshengec.com:1024": 5, + "thinkphp.com": 5, + "tiddlywiki5.com": 5, + "trac.dojotoolkit.org": 5, + "tracker.ceph.com": 5, + "tree-kit.com": 5, + "unomi.apache.org.": 5, + "ureport.com": 5, + "users.encs.concordia.ca": 5, + "vsintelli.com": 5, + "wiki.apache.org": 5, + "wiki.rpath.com": 5, + "andmp.com": 5, + "benjaminfleischer.com": 5, + "bigdiao.cc": 5, + "blcat.cn": 5, + "brocade.com": 5, + "conviso.com.br": 5, + "coreftp.com": 5, + "cs.utexas.edu": 5, + "dest-unreach.org": 5, + "egroupware.org": 5, + "elasticsearch.com": 5, + "fetchmail.info": 5, + "fomori.org": 5, + "getsymphony.com": 5, + "gevent.org": 5, + "graphicsmagick.org": 5, + "hackersb.cn": 5, + "inspircd.org": 5, + "isg.rhul.ac.uk": 5, + "ja-sig.org": 5, + "jcraft.com": 5, + "jianshu.com": 5, + "linuxgrill.com": 5, + "mbsd.jp": 5, + "mega-nerd.com": 5, + "movabletype.org": 5, + "mpxj.org": 5, + "nemux.org": 5, + "netbytesec.com": 5, + "opera.com": 5, + "pip-installer.org": 5, + "procheckup.com": 5, + "rabbitmq.com": 5, + "roothc.com.br": 5, + "rootlabs.com.br": 5, + "rsyslog.com": 5, + "securityweek.com": 5, + "slideshare.net": 5, + "sqlalchemy.org": 5, + "square16.org": 5, + "stack.nl": 5, + "tinc-vpn.org": 5, + "tripwire.com": 5, + "uzbl.org": 5, + "zeroscience.mk": 5, + "xmlsoft.org": 5, + "xxl-job.com": 5, + "yehg.net": 5, + "yourls.org": 5, + "zalando.com": 5, + "sony.com": 5, + "strawberry.rocks": 5, + "werewolves.world": 5, + "newegg.com": 5, + "consumer.huawei.com": 5, + "kb.parallels.com": 5, + "support.google.com": 5, + "community.targit.com": 5, + "unisoc.com": 5, + "support.broadcom.com": 5, + "security-portal.versa-networks.com": 5, + "sliderrevolution.com": 5, + "autodesk.com": 5, + "trust.mi.com": 5, + "git.proxmox.com": 5, + "docs.azul.com": 5, + "kibty.town": 5, + "flowus.cn": 5, + "wp-osm-plugin.hyumika.com": 5, + "nokia.com": 5, + "bdosecurity.de": 5, + "khronokernel.com": 5, + "wiki.securepoint.de": 5, + "bridewell.com": 5, + "arc.net": 5, + "bravurasecurity.com": 5, + "chiggerlor.substack.com": 5, + "csflabs.github.io": 5, + "manageengine.com": 5, + "zoho.com": 5, + "devolutions.net": 5, + "technitium.com": 5, + "dell.com": 5, + "eviden.com": 5, + "ptzoptics.com": 5, + "security.freebsd.org": 5, + "giflib.com": 5, + "support.hp.com": 5, + "icecms.com": 5, + "cyberdanube.com": 5, + "customers.codesys.com": 5, + "kaiten.ru": 5, + "keyence.com": 5, + "trustedcare.entrust.com": 5, + "codefactor.io": 5, + "codeastro.com": 5, + "exchange.checkmk.com": 5, + "cs-cart.com": 5, + "usvn.info": 5, + "takex-eng.co.jp": 5, + "ti.qianxin.com": 5, + "stuxxn.github.io": 5, + "trevorkems.com": 5, + "cve.mahi.be": 5, + "0xmupa.github.io": 5, + "forum.proxmox.com": 5, + "puredata.info": 5, + "docs.rocket.chat": 5, + "cirosec.de": 5, + "docs.adacore.com": 5, + "forums.ivanti.com": 5, + "thewatch.centreon.com": 5, + "papercut.com": 5, + "sophos.com": 5, + "issue-tracker.miraheze.org": 5, + "tiptel.com": 5, + "e-tax.nta.go.jp": 5, + "teamviewer.com": 5, + "answers.webroot.com": 5, + "cert.vde.com": 5, + "support.bull.com": 5, + "cnnvd.org.cn": 5, + "westerndigital.com": 5, + "closed-loop.biz": 5, + "entrust.com": 5, + "solvait.com": 5, + "woodwing.com": 5, + "blog.hawktesters.com": 5, + "zyxel.com": 5, + "0xahmed.ninja": 5, + "2012.appsec-forum.ch": 5, + "acme.com": 5, + "aliyundrive-webdav.com": 5, + "als.regnet.cz": 5, + "aluigi.altervista.org": 5, + "antirez.com": 5, + "apt.inguza.net": 5, + "arthurdejong.org": 5, + "asfws12.files.wordpress.com": 5, + "askubuntu.com": 5, + "aspell.net": 5, + "badlock.org": 5, + "baraktawily.blogspot.com": 5, + "benmmurphy.github.io": 5, + "bh.ht.vc": 5, + "bird.network.cz": 5, + "bk.ntp.org": 5, + "blog.addepar.com": 5, + "blog.c22.cc": 5, + "blog.chromium.org": 5, + "blog.dscpl.com.au": 5, + "blog.fortify.com": 5, + "blog.fuseyism.com": 5, + "blog.gmane.org": 5, + "blog.g-sec.lu": 5, + "blog.guya.net": 5, + "blog.infosectcbr.com.au": 5, + "blog.kazuhooku.com": 5, + "blog.koehntopp.de": 5, + "blog.nibblesec.org": 5, + "blog.noobroot.com": 5, + "blog.o0o.nu": 5, + "blog.opensecurityresearch.com": 5, + "blog.pear.php.net": 5, + "blog.pi3.com.pl": 5, + "blog.prosody.im": 5, + "blog.python.org": 5, + "blogs.adobe.com": 5, + "blog.scrt.ch": 5, + "blog.securitymouse.com": 5, + "blog.sendsafely.com": 5, + "blog.senr.io": 5, + "blogs.iss.net": 5, + "blogs.oracle.com": 5, + "blog.spiderlabs.com": 5, + "blogs.sun.com": 5, + "blog.swiecki.net": 5, + "blog.swiftmailer.org": 5, + "blog.topsec.com.cn": 5, + "boinc.berkeley.edu": 5, + "bugs.dokuwiki.org": 5, + "bugs.gw.com": 5, + "bugs.icu-project.org": 5, + "bugs.java.com": 5, + "bugs.ntp.org": 5, + "bugs.openttd.org": 5, + "bugzilla.maptools.org": 5, + "buzz.typo3.org": 5, + "cakeforge.org": 5, + "cat.eyalro.net": 5, + "checkpw.sourceforge.net": 5, + "cisofy.com": 5, + "clicky.me": 5, + "co3k.org": 5, + "code.fabfile.org": 5, + "code.qt.io": 5, + "community.igniterealtime.org": 5, + "crypto.junod.info": 5, + "csrf.htmlpurifier.org": 5, + "cve.circl.lu": 5, + "cvs.sourceforge.net": 5, + "cybersecurity.upv.es": 5, + "danlec.com": 5, + "dev.exiv2.org": 5, + "dev.mutt.org": 5, + "dev.plone.org": 5, + "dev.subrion.org": 5, + "distro.conectiva.com.br": 5, + "docs.sequelizejs.com": 5, + "download.opensuse.org": 5, + "download.osgeo.org": 5, + "download.savannah.gnu.org": 5, + "download.strongswan.org": 5, + "drosenbe.blogspot.com": 5, + "drupalcode.org": 5, + "dsecrg.com": 5, + "em386.blogspot.com": 5, + "everdox.net": 5, + "fastcompression.blogspot.fr": 5, + "flash.flowplayer.org": 5, + "forum.xda-developers.com": 5, + "freeciv.wikia.com": 5, + "ftp.gnu.org": 5, + "ftp.naist.jp": 5, + "ftp.netbsd.org": 5, + "ftp.NetBSD.org": 5, + "gaganpreet.in": 5, + "ganglia.info": 5, + "geojson.org": 5, + "getid3.sourceforge.net": 5, + "git-blame.blogspot.com.es": 5, + "git.cyrusimap.org": 5, + "git.gluster.org": 5, + "git.hylafax.org": 5, + "git.infradead.org": 5, + "git.kernel.dk": 5, + "git.libav.org": 5, + "git.lxde.org": 5, + "git.mathias-kettner.de": 5, + "git.nordu.net": 5, + "gitorious.org": 5, + "git.savannah.nongnu.org": 5, + "git.tuxfamily.org": 5, + "googlechromereleases.blogspot.co.uk": 5, + "gridscheduler.sourceforge.net": 5, + "guac-dev.org": 5, + "h30499.www3.hp.com": 5, + "habrahabr.ru": 5, + "hac425.unaux.com": 5, + "ha.xxor.se": 5, + "hg.libsdl.org": 5, + "hg.savannah.gnu.org": 5, + "hkpco.kr": 5, + "homakov.blogspot.com": 5, + "icedtea.classpath.org": 5, + "inertiawar.com": 5, + "invisible-island.net": 5, + "ircrash.com": 5, + "isecpartners.github.io": 5, + "jaanuskp.blogspot.com": 5, + "jgarber.lighthouseapp.com": 5, + "jira.opensymphony.com": 5, + "jira.ow2.org": 5, + "juliusdavies.ca": 5, + "k3research.outerhaven.de": 5, + "kbase.redhat.com": 5, + "kerneltrap.org": 5, + "klikki.fi": 5, + "knoxin.blogspot.co.uk": 5, + "krbdev.mit.edu": 5, + "lab.cs.ttu.ee": 5, + "lcamtuf.blogspot.com.au": 5, + "ledgersmbdev.blogspot.ca": 5, + "libcloud.apache.org": 5, + "libexif.sourceforge.net": 5, + "linuxtesting.org": 5, + "linuxtv.org": 5, + "listengine.tuxfamily.org": 5, + "lists.alioth.debian.org": 5, + "lists.busybox.net": 5, + "lists.clamav.net": 5, + "lists.fusionforge.org": 5, + "lists.infradead.org": 5, + "lists.llvm.org": 5, + "lists.ntp.org": 5, + "lists.openvz.org": 5, + "lists.owasp.org": 5, + "lists.unbit.it": 5, + "lists.vmware.com": 5, + "lists.xiph.org": 5, + "lists.xymon.com": 5, + "lucene.apache.org": 5, + "lustre.org": 5, + "mahara.org": 5, + "mail.globnix.net": 5, + "mail.jabber.org": 5, + "mail.kde.org": 5, + "mailman.alsa-project.org": 5, + "mailman.mit.edu": 5, + "mail.openjdk.java.net": 5, + "mails.dpdk.org": 5, + "malloc.im": 5, + "martin.swende.se": 5, + "mathias-kettner.de": 5, + "mayaa.seasar.org": 5, + "metadata.ftp-master.debian.org": 5, + "mirror.easyname.at": 5, + "mirror.linux.org.au": 5, + "mod-security.svn.sourceforge.net": 5, + "modwsgi.readthedocs.org": 5, + "moinmoin.wikiwikiweb.de": 5, + "mov.sx": 5, + "mstrokin.com": 5, + "mysqlblog.fivefarmers.com": 5, + "netatalk.sourceforge.net": 5, + "net-ninja-mr.me": 5, + "news.dieweltistgarnichtso.net": 5, + "news.tryton.org": 5, + "nmav.gnutls.org": 5, + "nongnu.askapache.com": 5, + "old.blog.phusion.nl": 5, + "openjpeg.googlecode.com": 5, + "openvswitch.org": 5, + "osandamalith.wordpress.com": 5, + "osdir.com": 5, + "oss.sgi.com": 5, + "otiose.dhs.org": 5, + "owasp-esapi-java.googlecode.com": 5, + "owasp-java-html-sanitizer.googlecode.com": 5, + "ozlabs.org": 5, + "packetstormsecurity.nl": 5, + "pastie.caboo.se": 5, + "penturalabs.wordpress.com": 5, + "people.debian.org": 5, + "perception-point.io": 5, + "perl5.git.perl.org": 5, + "phpunit.vulnbusters.com": 5, + "pillow.readthedocs.org": 5, + "pmt.sourceforge.net": 5, + "post-office.corp.redhat.com": 5, + "proftpd.org": 5, + "projects.edgewall.com": 5, + "pseudo-flaw.net": 5, + "pylonshq.com": 5, + "pyropus.ca": 5, + "pywebdav.googlecode.com": 5, + "quassel-irc.org": 5, + "quickgit.kde.org": 5, + "qwertwwwe.github.io": 5, + "r00tin.blogspot.com": 5, + "rabbit.dereferenced.org": 5, + "rack.github.com": 5, + "railspikes.com": 5, + "restlet.org": 5, + "riddle.link": 5, + "rosariosis.com": 5, + "roundup.cvs.sourceforge.net": 5, + "rubysec.github.io": 5, + "01.org": 5, + "0day.work": 5, + "s1m0n.dft-labs.eu": 5, + "admin.hostpoint.ch": 5, + "alioth-lists.debian.net": 5, + "android-review.googlesource.com": 5, + "anotepad.com": 5, + "antichat.com": 5, + "appgateresearch.blogspot.com": 5, + "archive.apache.org": 5, + "artifex.com": 5, + "attachments.samba.org": 5, + "attic.apache.org": 5, + "bertjwregeer.keybase.pub": 5, + "bitslog.com": 5, + "blade.tencent.com": 5, + "blog.cloudpassage.com": 5, + "blog.convisoappsec.com": 5, + "blog.cryptographyengineering.com": 5, + "blog.documentfoundation.org": 5, + "blog.hartwork.org": 5, + "blog.hboeck.de": 5, + "blog.mozilla.org": 5, + "blog.qt.io": 5, + "blog.quarkslab.com": 5, + "blog.smarttecs.com": 5, + "blog.sqreen.com": 5, + "blog.wpscan.org": 5, + "bnbdr.github.io": 5, + "botan.randombit.net": 5, + "bpmn.io": 5, + "bro-tracker.atlassian.net": 5, + "bugreports.qt.io": 5, + "bugs.exim.org": 5, + "bugs.gnunet.org": 5, + "bugs.horde.org": 5, + "bugs.linuxfoundation.org": 5, + "bugs.openvz.org": 5, + "bugs.webkit.org": 5, + "bugzilla.clamav.net": 5, + "bugzilla.mindrot.org": 5, + "bugzilla.samba.org": 5, + "bugzilla.tianocore.org": 5, + "busybox.net": 5, + "c-ares.haxx.se": 5, + "ceph.io": 5, + "cfreal.github.io": 5, + "chromiumcodereview.appspot.com": 5, + "civicrm.org": 5, + "codechecker-demo.eastus.cloudapp.azure.com": 5, + "code.launchpad.net": 5, + "codereview.appspot.com": 5, + "codesearch.debian.net": 5, + "community.jboss.org": 5, + "community.qualys.com": 5, + "community.sophos.com": 5, + "contrib.spip.net": 5, + "coreymhudson.github.io": 5, + "crashes.fuzzing-project.org": 5, + "crrev.com": 5, + "crypto.stanford.edu": 5, + "defuse.ca": 5, + "deshal3v.github.io": 5, + "developer.atlassian.com": 5, + "dev.gnupg.org": 5, + "dev.recurly.com": 5, + "discourse.aurelia.io": 5, + "discourse.gnome.org": 5, + "discuss.gradle.org": 5, + "discuss.neos.io": 5, + "docs.inspircd.org": 5, + "dom4j.github.io": 5, + "donncha.is": 5, + "download.igniterealtime.org": 5, + "dyntopia.com": 5, + "secondlookforensics.com": 5, + "secureappdev.blogspot.com": 5, + "secureyourit.co.uk": 5, + "security.cucumberlinux.com": 5, + "edk2-docs.gitbook.io": 5, + "edk2.groups.io": 5, + "edoverflow.com": 5, + "efail.de": 5, + "elixir.bootlin.com": 5, + "en.bitcoinwiki.org": 5, + "enigmail.net": 5, + "esnet-security.github.io": 5, + "exploitbox.io": 5, + "eyalitkin.wordpress.com": 5, + "fakhrizulkifli.github.io": 5, + "fastcompression.blogspot.ca": 5, + "fastd.readthedocs.io": 5, + "feh.finalrewind.org": 5, + "fisheye.codehaus.org": 5, + "forums.grsecurity.net": 5, + "forum.snapcraft.io": 5, + "forums.servicestack.net": 5, + "francozappa.github.io": 5, + "sf.snu.ac.kr": 5, + "fuelphp.com": 5, + "gaffer.ptitcanardnoir.org": 5, + "gerrit.googlesource.com": 5, + "git.busybox.net": 5, + "git.centos.org": 5, + "git.entrouvert.org": 5, + "git.gnunet.org": 5, + "git.gnupg.org": 5, + "github.blog": 5, + "githubengineering.com": 5, + "gitlab.xiph.org": 5, + "git.netfilter.org": 5, + "git.tartarus.org": 5, + "git.tt-rss.org": 5, + "git.whamcloud.com": 5, + "git.zabbix.com": 5, + "gnats.netbsd.org": 5, + "gnupg.org": 5, + "googleonlinesecurity.blogspot.com": 5, + "googleprojectzero.blogspot.co.uk": 5, + "googleprojectzero.blogspot.cz": 5, + "gruss.cc": 5, + "guidovranken.files.wordpress.com": 5, + "gusralph.info": 5, + "h20564.www2.hpe.com": 5, + "h20566.www2.hp.com": 5, + "heasarc.gsfc.nasa.gov": 5, + "hg.ucc.asn.au": 5, + "shibboleth.internet2.edu": 5, + "hosein-vita.medium.com": 5, + "httpoxy.org": 5, + "icepng.github.io": 5, + "icinga.com": 5, + "insert-script.blogspot.com": 5, + "insights.sei.cmu.edu": 5, + "insights.ubuntu.com": 5, + "invent.kde.org": 5, + "ioquake3.org": 5, + "issues.civicrm.org": 5, + "issues.gerritcodereview.com": 5, + "issuetracker.google.com": 5, + "site.icu-project.org": 5, + "site.pi3.com.pl": 5, + "jeffhacks.com": 5, + "kate.io": 5, + "kb.bluecoat.com": 5, + "kingcope.wordpress.com": 5, + "knobattack.com": 5, + "komodoplatform.com": 5, + "leastauthority.com": 5, + "libexpat.github.io": 5, + "libosinfo.org": 5, + "libreswan.org": 5, + "lists.ath9k.org": 5, + "lists.clusterlabs.org": 5, + "lists.cypherpunks.ca": 5, + "lists.iai.uni-bonn.de": 5, + "lists.immunityinc.com": 5, + "lists.isc.org": 5, + "lists.linuxcontainers.org": 5, + "lists.nic.cz": 5, + "lists.openldap.org": 5, + "lists.open-mesh.org": 5, + "lists.osuosl.org": 5, + "lists.sequoia-pgp.org": 5, + "lists.tartarus.org": 5, + "lists.ubuntu.com": 5, + "lock.cmpxchg8b.com": 5, + "mailarchives.bentasker.co.uk": 5, + "mail.gnu.org": 5, + "mailman-eng.corp.redhat.com": 5, + "make.wordpress.org": 5, + "mariadb.atlassian.net": 5, + "marlam.de": 5, + "smarty-php.googlecode.com": 5, + "maustin.net": 5, + "mcabber.com": 5, + "mega.nz": 5, + "meltdownattack.com": 5, + "micahflee.com": 5, + "mirror.fail": 5, + "nathandavison.com": 5, + "neomutt.org": 5, + "neopg.io": 5, + "nethack.org": 5, + "newrelic.com": 5, + "nohats.ca": 5, + "snoopy.cvs.sourceforge.net": 5, + "nvisium.com": 5, + "old.reddit.com": 5, + "open-docs.neuvector.com": 5, + "openmpt.org": 5, + "openssf.slack.com": 5, + "openssl-library.org": 5, + "opnsec.com": 5, + "orpheus-lyre.info": 5, + "osdn.net": 5, + "oss.clusterlabs.org": 5, + "os-s.net": 5, + "ostif.org": 5, + "source.jboss.org": 5, + "packages.qa.debian.org": 5, + "paste.pound-python.org": 5, + "patchwork.ffmpeg.org": 5, + "patchwork.freedesktop.org": 5, + "people.csail.mit.edu": 5, + "people.fedoraproject.org": 5, + "phabricator.kde.org": 5, + "philwantsfish.github.io": 5, + "spl0it.org": 5, + "platypusattack.com": 5, + "postlister.uninett.no": 5, + "projects.gnome.org": 5, + "projects.ow2.org": 5, + "pyyaml.docsforge.com": 5, + "qa.debian.org": 5, + "qtpass.org": 5, + "rachelbythebay.com": 5, + "raw.github.com": 5, + "raw.globalsecuritydatabase.org": 5, + "rdot.org": 5, + "relistan.com": 5, + "robots.thoughtbot.com": 5, + "rtpbleed.com": 5, + "rt.perl.org": 5, + "ruhr-uni-bochum.sciebo.de": 5, + "scannell.me": 5, + "scarybeastsecurity.blogspot.ch": 5, + "scarybeastsecurity.blogspot.com": 5, + "scarybeastsecurity.blogspot.dk": 5, + "scumjr.github.io": 5, + "secure.phabricator.com": 5, + "secure.php.net": 5, + "security-center.intel.com": 5, + "serenity.is": 5, + "sick.codes": 5, + "sitewat.ch": 5, + "smacktls.com": 5, + "source.openmpt.org": 5, + "spectreattack.com": 5, + "speirofr.appspot.com": 5, + "ssrg.nicta.com.au": 5, + "support.cloud.engineyard.com": 5, + "support.process-one.net": 5, + "svn.filezilla-project.org": 5, + "svn.resiprocate.org": 5, + "sweet32.info": 5, + "swiftstack.com": 5, + "sword.bladex.cn": 5, + "sympa.inria.fr": 5, + "t2.fi": 5, + "tapestry.apache.org": 5, + "thejh.net": 5, + "therecord.media": 5, + "threatpost.com": 5, + "tomforb.es": 5, + "trac.gajim.org": 5, + "tracker.ardour.org": 5, + "trac.mplayerhq.hu": 5, + "trac.nginx.org": 5, + "tt-rss.org": 5, + "unicode-org.atlassian.net": 5, + "unix.stackexchange.com": 5, + "utcc.utoronto.ca": 5, + "vishnudevtj.github.io": 5, + "vndh.net": 5, + "svn.ec-cube.net": 5, + "svn.haxx.se": 5, + "svn.ruby-lang.org": 5, + "svn.tartarus.org": 5, + "vulnhive.com": 5, + "web-in-security.blogspot.ca": 5, + "wid.cert-bund.de": 5, + "wiki.openvz.org": 5, + "wiki.samba.org": 5, + "worthdoingbadly.com": 5, + "wpa3.mathyvanhoef.com": 5, + "131002.net": 5, + "www3.sqlite.org": 5, + "www3.trustwave.com": 5, + "acunetix.com": 5, + "alphabot.com": 5, + "amd.com": 5, + "arista.com": 5, + "armis.com": 5, + "bamsoftware.com": 5, + "bitdefender.com": 5, + "cabextract.org.uk": 5, + "cazzulino.com": 5, + "cisecurity.org": 5, + "cl.cam.ac.uk": 5, + "cncf.io": 5, + "contextis.com": 5, + "cubeyond.net": 5, + "davical.org": 5, + "eclypsium.com": 5, + "evonide.com": 5, + "exiv2.org": 5, + "flexera.com": 5, + "freeplane.org": 5, + "futureweb.at": 5, + "gnu.org": 5, + "icinga.org": 5, + "inputzero.io": 5, + "intezer.com": 5, + "java.com": 5, + "jsof-tech.com": 5, + "jwz.org": 5, + "kernelmode.blog": 5, + "krackattacks.com": 5, + "kvakil.me": 5, + "ldap-account-manager.org": 5, + "legacysecuritygroup.com": 5, + "libssh2.org": 5, + "lsexperts.de": 5, + "mehmetince.net": 5, + "midnight-commander.org": 5, + "nikhef.nl": 5, + "npmjs.org": 5, + "openldap.org": 5, + "percona.com": 5, + "portcullis-security.com": 5, + "pureftpd.org": 5, + "python.org": 5, + "qbittorrent.org": 5, + "redteam-pentesting.de": 5, + "reversinglabs.com": 5, + "rootshellsecurity.net": 5, + "rorsecurity.info": 5, + "saddns.net": 5, + "secfu.net": 5, + "secura.com": 5, + "seebug.org": 5, + "sentinelone.com": 5, + "sigsac.org": 5, + "stunnel.org": 5, + "teskalabs.com": 5, + "theregister.co.uk": 5, + "thezdi.com": 5, + "timmclean.net": 5, + "trendmicro.com": 5, + "vice.com": 5, + "vidocsecurity.com": 5, + "viestintavirasto.fi": 5, + "virustotal.com": 5, + "vusec.net": 5, + "xairy.github.io": 5, + "sysoev.ru": 5, + "t3.dotgnu.info": 5, + "tartarus.org": 5, + "taviso.decsystem.org": 5, + "telia.dl.sourceforge.net": 5, + "telussecuritylabs.com": 5, + "thekelleys.org.uk": 5, + "tickets.opscode.com": 5, + "trac.kodi.tv": 5, + "trac.transmissionbt.com": 5, + "vcs.pcre.org": 5, + "venom.crowdstrike.com": 5, + "vladz.devzero.fr": 5, + "webservsec.blogspot.com": 5, + "wiki2.dovecot.org": 5, + "wiki.audacityteam.org": 5, + "wiki.lustre.org": 5, + "wiki.postgresql.org": 5, + "worldofpadman.com": 5, + "www13.itrc.hp.com": 5, + "agarri.fr": 5, + "agrs.tu-berlin.de": 5, + "apsis.ch": 5, + "argyllcms.com": 5, + "arm.linux.org.uk": 5, + "asmail.be": 5, + "attrition.org": 5, + "bacula.org": 5, + "betanews.com": 5, + "binarysniper.net": 5, + "bugzilla.org": 5, + "cc.gatech.edu": 5, + "cert.fi": 5, + "ciac.org": 5, + "cisco.com": 5, + "codenomicon.com": 5, + "collabtive.o-dyn.de": 5, + "communities.hp.com": 5, + "cs.bu.edu": 5, + "cs.tau.ac.IL": 5, + "cs.tau.ac.il": 5, + "cs.technion.ac.il": 5, + "dotnetnuke.com": 5, + "droid-life.com": 5, + "droidrzr.com": 5, + "droidsec.org": 5, + "educatedguesswork.org": 5, + "elasticsearch.org": 5, + "elsherei.com": 5, + "eucalyptus.com": 5, + "evernote.com": 5, + "exploringbinary.com": 5, + "firebirdsql.org": 5, + "floyd.ch": 5, + "freelists.org": 5, + "freerdp.com": 5, + "gdssecurity.com": 5, + "getchef.com": 5, + "gulftech.org": 5, + "gwtproject.org": 5, + "haproxy.org": 5, + "hdwsec.fr": 5, + "hitachi-support.com": 5, + "htbridge.ch": 5, + "ijg.org": 5, + "infoq.com": 5, + "infradead.org": 5, + "ingate.com": 5, + "intelsecurity.com": 5, + "jbkempf.com": 5, + "jplayer.org": 5, + "kernelhub.org": 5, + "konakart.com": 5, + "libraw.org": 5, + "live555.com": 5, + "lua.org": 5, + "mandrakesecure.net": 5, + "maradns.org": 5, + "mathyvanhoef.com": 5, + "metasploit.com": 5, + "mh-sec.de": 5, + "mindrot.org": 5, + "mitls.org": 5, + "mongodb.org": 5, + "mono-project.com": 5, + "nds.rub.de": 5, + "networkworld.com": 5, + "nntp.perl.org": 5, + "nruns.com": 5, + "nsfocus.net": 5, + "oliverkarow.de": 5, + "opencms.org": 5, + "openoffice.org": 5, + "palemoon.org": 5, + "paul-moore.com": 5, + "phenoelit.org": 5, + "pimcore.org": 5, + "pnigos.com": 5, + "process-one.net": 5, + "quantumleap.it": 5, + "rafayhackingarticles.net": 5, + "rajatswarup.com": 5, + "ratbox.org": 5, + "rul3z.de": 5, + "sbosnet.nl": 5, + "securation.com": 5, + "securegoose.org": 5, + "securiteam.com": 5, + "sendmail.com": 5, + "simplesystems.org": 5, + "slimframework.com": 5, + "sogo.nu": 5, + "splitbrain.org": 5, + "springsource.com": 5, + "squirrelmail.org": 5, + "supercluster.org": 5, + "synacktiv.ninja": 5, + "tedunangst.com": 5, + "tele-consulting.com": 5, + "tombom.co.uk": 5, + "tt-forums.net": 5, + "undeadly.org": 5, + "unixodbc.org": 5, + "x.org": 5, + "zweitag.de": 5, + "xstream.codehaus.org": 5, + "xteam.baidu.com": 5, + "xync.org": 5, + "zsh.sourceforge.net": 5, + "project-zero.issues.chromium.org": 5, + "anker.com": 5, + "visionspace.com": 5, + "eufy.com": 5, + "advisory-inbox.githubapp.com": 5, + "myoffice.ru": 5, + "support.myoffice.ru": 5, + "planex.co.jp": 5, + "docs.iredmail.org": 5, + "iredmail.org": 5 } \ No newline at end of file From 71a2a926bacb47055e87b4747fae54f253604fab Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 7 Nov 2024 22:45:54 +0530 Subject: [PATCH 18/20] Add license header Signed-off-by: Keshav Priyadarshi --- vulnerabilities/pipelines/compute_package_risk.py | 9 +++++++++ vulnerabilities/pipelines/enhance_with_exploitdb.py | 9 +++++++++ vulnerabilities/pipelines/enhance_with_kev.py | 9 +++++++++ vulnerabilities/pipelines/enhance_with_metasploit.py | 9 +++++++++ .../tests/pipelines/test_compute_package_risk.py | 9 +++++++++ ...{test_exploitdb.py => test_enhance_with_exploitdb.py} | 9 +++++++++ .../pipelines/{test_kev.py => test_enhance_with_kev.py} | 9 +++++++++ ...est_metasploit.py => test_enhance_with_metasploit.py} | 9 +++++++++ .../tests/pipelines/test_pysec_importer_pipeline.py | 1 + vulnerabilities/tests/test_risk.py | 9 +++++++++ 10 files changed, 82 insertions(+) rename vulnerabilities/tests/pipelines/{test_exploitdb.py => test_enhance_with_exploitdb.py} (75%) rename vulnerabilities/tests/pipelines/{test_kev.py => test_enhance_with_kev.py} (75%) rename vulnerabilities/tests/pipelines/{test_metasploit.py => test_enhance_with_metasploit.py} (76%) diff --git a/vulnerabilities/pipelines/compute_package_risk.py b/vulnerabilities/pipelines/compute_package_risk.py index 525da599d..d064f4597 100644 --- a/vulnerabilities/pipelines/compute_package_risk.py +++ b/vulnerabilities/pipelines/compute_package_risk.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + from aboutcode.pipeline import LoopProgress from vulnerabilities.models import Package diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py index 54554f951..b31ac4134 100644 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import csv import io import logging diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index 6372bd3b0..d8244944a 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import logging from traceback import format_exc as traceback_format_exc diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index 72897abd0..d6989ab47 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import logging from traceback import format_exc as traceback_format_exc diff --git a/vulnerabilities/tests/pipelines/test_compute_package_risk.py b/vulnerabilities/tests/pipelines/test_compute_package_risk.py index e44fa424a..7c197e812 100644 --- a/vulnerabilities/tests/pipelines/test_compute_package_risk.py +++ b/vulnerabilities/tests/pipelines/test_compute_package_risk.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import pytest from vulnerabilities.models import AffectedByPackageRelatedVulnerability diff --git a/vulnerabilities/tests/pipelines/test_exploitdb.py b/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py similarity index 75% rename from vulnerabilities/tests/pipelines/test_exploitdb.py rename to vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py index f08f7fec0..f54dad55d 100644 --- a/vulnerabilities/tests/pipelines/test_exploitdb.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import os from unittest import mock from unittest.mock import Mock diff --git a/vulnerabilities/tests/pipelines/test_kev.py b/vulnerabilities/tests/pipelines/test_enhance_with_kev.py similarity index 75% rename from vulnerabilities/tests/pipelines/test_kev.py rename to vulnerabilities/tests/pipelines/test_enhance_with_kev.py index 71583a617..a93c16555 100644 --- a/vulnerabilities/tests/pipelines/test_kev.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_kev.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import os from unittest import mock from unittest.mock import Mock diff --git a/vulnerabilities/tests/pipelines/test_metasploit.py b/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py similarity index 76% rename from vulnerabilities/tests/pipelines/test_metasploit.py rename to vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py index 1116950d2..eea99e0ca 100644 --- a/vulnerabilities/tests/pipelines/test_metasploit.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import os from unittest import mock from unittest.mock import Mock diff --git a/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py index b596c8f1d..c38256c01 100644 --- a/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py +++ b/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py @@ -6,6 +6,7 @@ # See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # + import json from pathlib import Path from unittest import TestCase diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py index 34fc5f397..96d9f9445 100644 --- a/vulnerabilities/tests/test_risk.py +++ b/vulnerabilities/tests/test_risk.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import pytest from vulnerabilities.models import Exploit From 3b33094fa51f816cd9d707dd14dc9c895ba2633e Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 7 Nov 2024 22:47:45 +0530 Subject: [PATCH 19/20] Move weight config to python file Signed-off-by: Keshav Priyadarshi --- vulnerabilities/risk.py | 15 +++++++++++---- .../weight_config.py | 15 ++++++++++++--- 2 files changed, 23 insertions(+), 7 deletions(-) rename weight_config.json => vulnerabilities/weight_config.py (99%) diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index 815435136..9eb4ac6ec 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -1,4 +1,13 @@ -from pathlib import Path +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + from urllib.parse import urlparse from vulnerabilities.models import AffectedByPackageRelatedVulnerability @@ -7,11 +16,9 @@ from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.severity_systems import EPSS -from vulnerabilities.utils import load_json +from vulnerabilities.weight_config import WEIGHT_CONFIG DEFAULT_WEIGHT = 5 -WEIGHT_CONFIG_PATH = Path(__file__).parent.parent / "weight_config.json" -WEIGHT_CONFIG = load_json(WEIGHT_CONFIG_PATH) def get_weighted_severity(severities): diff --git a/weight_config.json b/vulnerabilities/weight_config.py similarity index 99% rename from weight_config.json rename to vulnerabilities/weight_config.py index f6cd83349..f9ad8292d 100644 --- a/weight_config.json +++ b/vulnerabilities/weight_config.py @@ -1,4 +1,13 @@ -{ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +WEIGHT_CONFIG = { "nvd.nist.gov": 9, "api.first.org": 9, "github.com": 9, @@ -2906,5 +2915,5 @@ "support.myoffice.ru": 5, "planex.co.jp": 5, "docs.iredmail.org": 5, - "iredmail.org": 5 -} \ No newline at end of file + "iredmail.org": 5, +} From c3d10a7fc5373461accf6872734fff5b69e7b5b4 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 7 Nov 2024 23:20:01 +0530 Subject: [PATCH 20/20] Skip packages with no risk score Signed-off-by: Keshav Priyadarshi --- vulnerabilities/pipelines/compute_package_risk.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/pipelines/compute_package_risk.py b/vulnerabilities/pipelines/compute_package_risk.py index d064f4597..e5b48ea0e 100644 --- a/vulnerabilities/pipelines/compute_package_risk.py +++ b/vulnerabilities/pipelines/compute_package_risk.py @@ -16,7 +16,9 @@ class ComputePackageRiskPipeline(VulnerableCodePipeline): """ - Risk Assessment Pipeline for Package Vulnerabilities: Iterate through the packages and evaluate their associated risk. + Compute risk score for packages. + + See https://github.com/aboutcode-org/vulnerablecode/issues/1543 """ pipeline_id = "compute_package_risk" @@ -41,6 +43,10 @@ def add_package_risk_score(self): for package in progress.iter(affected_packages.paginated()): risk_score = compute_package_risk(package) + + if not risk_score: + continue + package.risk_score = risk_score updatables.append(package)