From 3cee7717864c54c50b865cefc7d6c18d7a8783b7 Mon Sep 17 00:00:00 2001 From: Tushar Goel <34160672+TG1999@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:15:17 +0530 Subject: [PATCH] Add indexes for models (#1701) * Reorder Meta and core properties Signed-off-by: Tushar Goel * Add todo Signed-off-by: Tushar Goel * Add indexes on models Signed-off-by: Tushar Goel * Add migrations Signed-off-by: Tushar Goel * Add indexes Signed-off-by: Tushar Goel * Add migrations Signed-off-by: Tushar Goel * Fix formatting issues Signed-off-by: Tushar Goel --------- Signed-off-by: Tushar Goel --- ...ost_alter_package_version_rank_and_more.py | 83 +++++++++++++++++++ vulnerabilities/models.py | 44 +++++++--- 2 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 vulnerabilities/migrations/0085_alter_package_is_ghost_alter_package_version_rank_and_more.py diff --git a/vulnerabilities/migrations/0085_alter_package_is_ghost_alter_package_version_rank_and_more.py b/vulnerabilities/migrations/0085_alter_package_is_ghost_alter_package_version_rank_and_more.py new file mode 100644 index 000000000..1b1181179 --- /dev/null +++ b/vulnerabilities/migrations/0085_alter_package_is_ghost_alter_package_version_rank_and_more.py @@ -0,0 +1,83 @@ +# Generated by Django 4.2.16 on 2024-12-18 10:09 + +import aboutcode.hashid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0084_alter_package_options_package_version_rank"), + ] + + operations = [ + migrations.AlterField( + model_name="package", + name="is_ghost", + field=models.BooleanField( + db_index=True, + default=False, + help_text="True if the package does not exist in the upstream package manager or its repository.", + ), + ), + migrations.AlterField( + model_name="package", + name="version_rank", + field=models.IntegerField( + db_index=True, + default=0, + help_text="Rank of the version to support ordering by version. Rank zero means the rank has not been defined yet", + ), + ), + migrations.AlterField( + model_name="vulnerability", + name="vulnerability_id", + field=models.CharField( + blank=True, + db_index=True, + default=aboutcode.hashid.build_vcid, + help_text="Unique identifier for a vulnerability in the external representation. It is prefixed with VCID-", + max_length=20, + unique=True, + ), + ), + migrations.AlterField( + model_name="vulnerabilityreference", + name="reference_id", + field=models.CharField( + blank=True, + db_index=True, + help_text="An optional reference ID, such as DSA-4465-1 when available", + max_length=200, + ), + ), + migrations.AlterField( + model_name="vulnerabilityseverity", + name="url", + field=models.URLField( + db_index=True, + help_text="URL to the vulnerability severity", + max_length=1024, + null=True, + ), + ), + migrations.AddIndex( + model_name="package", + index=models.Index( + fields=["type", "namespace", "name"], name="vulnerabili_type_825918_idx" + ), + ), + migrations.AddIndex( + model_name="package", + index=models.Index( + fields=["type", "namespace", "name", "qualifiers", "subpath"], + name="vulnerabili_type_8e6aff_idx", + ), + ), + migrations.AddIndex( + model_name="package", + index=models.Index( + fields=["type", "namespace", "name", "version"], name="vulnerabili_type_f6687a_idx" + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index c2e89022f..6248e1e47 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -161,6 +161,7 @@ class VulnerabilitySeverity(models.Model): max_length=1024, null=True, help_text="URL to the vulnerability severity", + db_index=True, ) scoring_system_choices = tuple( @@ -212,6 +213,7 @@ class Vulnerability(models.Model): default=utils.build_vcid, help_text="Unique identifier for a vulnerability in the external representation. " "It is prefixed with VCID-", + db_index=True, ) summary = models.TextField( @@ -453,6 +455,7 @@ class VulnerabilityReference(models.Model): max_length=200, help_text="An optional reference ID, such as DSA-4465-1 when available", blank=True, + db_index=True, ) objects = VulnerabilityReferenceQuerySet.as_manager() @@ -509,6 +512,7 @@ def get_fixed_by_package_versions(self, purl: PackageURL, fix=True): if fix: filter_dict["fixing_vulnerabilities__isnull"] = False + # TODO: why do we need distinct return Package.objects.filter(**filter_dict).distinct() def get_or_create_from_purl(self, purl: Union[PackageURL, str]): @@ -695,6 +699,7 @@ class Package(PackageURLMixin): is_ghost = models.BooleanField( default=False, help_text="True if the package does not exist in the upstream package manager or its repository.", + db_index=True, ) risk_score = models.DecimalField( @@ -709,10 +714,36 @@ class Package(PackageURLMixin): help_text="Rank of the version to support ordering by version. Rank " "zero means the rank has not been defined yet", default=0, + db_index=True, ) objects = PackageQuerySet.as_manager() + class Meta: + unique_together = ["type", "namespace", "name", "version", "qualifiers", "subpath"] + ordering = ["type", "namespace", "name", "version_rank", "version", "qualifiers", "subpath"] + indexes = [ + # Index for getting al versions of a package + models.Index(fields=["type", "namespace", "name"]), + models.Index(fields=["type", "namespace", "name", "qualifiers", "subpath"]), + # Index for getting a specific version of a package + models.Index( + fields=[ + "type", + "namespace", + "name", + "version", + ] + ), + ] + + def __str__(self): + return self.package_url + + @property + def purl(self): + return self.package_url + def save(self, *args, **kwargs): """ Save, normalizing PURL fields. @@ -738,17 +769,6 @@ def save(self, *args, **kwargs): self.plain_package_url = str(plain_purl) super().save(*args, **kwargs) - @property - def purl(self): - return self.package_url - - class Meta: - unique_together = ["type", "namespace", "name", "version", "qualifiers", "subpath"] - ordering = ["type", "namespace", "name", "version_rank", "version", "qualifiers", "subpath"] - - def __str__(self): - return self.package_url - @property def calculate_version_rank(self): """ @@ -981,12 +1001,14 @@ class PackageRelatedVulnerabilityBase(models.Model): package = models.ForeignKey( Package, on_delete=models.CASCADE, + db_index=True, # related_name="%(class)s_set", # Unique related_name per subclass ) vulnerability = models.ForeignKey( Vulnerability, on_delete=models.CASCADE, + db_index=True, # related_name="%(class)s_set", # Unique related_name per subclass )