Merge pull request #73 from cve-search/improved-user-profile

new: [website] Add new fields to User model

CHANGELOG.md

@@ -1,6 +1,48 @@
 Vulnerability Lookup Changelog
 ==============================
 
+## 1.6.0 (not yet released)
+
+### News
+
+- Importer for Tailscale vulnerabilities
+  [#68](https://github.com/cve-search/vulnerability-lookup/issues/68)
+- New user profile page with more information and detection of the country
+  during user sign-up using the CIRCL MMDB service
+  [#73](https://github.com/cve-search/vulnerability-lookup/pull/73)
+- Added the ability to filter comments by any taxonomy tags by clicking on the corresponding badge
+  ([b3e0bdf](https://github.com/cve-search/vulnerability-lookup/commit/b3e0bdf6e15b5821c3e7a4e574fa452857fd0410))
+- Implemented a function to back up the database using pg_dump. This function is automatically triggered by the
+  project's update command, ensuring a backup is created before any database upgrades take place.
+  ([75ee913](https://github.com/cve-search/vulnerability-lookup/commit/75ee9132beb9df1c5a847eb11488dd5241f3ee1f))
+
+
+### Improvements
+
+- [API] Enhanced detection of CVE, GHSA, and PySec IDs within bundle descriptions and comments.
+  This enables automatic identification of related vulnerabilities linked to a comment or a bundle.
+  ([2c00695](https://github.com/cve-search/vulnerability-lookup/commit/2c00695864dafc20de6394dc0fe7e3d02ff570f1),
+  [162a599](https://github.com/cve-search/vulnerability-lookup/commit/162a599ac12482e3cfb6e0d6d9d1566ef10dab01),
+  [401d780](https://github.com/cve-search/vulnerability-lookup/commit/401d78081f4aa6c14664a74aaf4e3c4398d05a33)).
+- Added more validation to the various attributes of the User model.
+  ([758e571](https://github.com/cve-search/vulnerability-lookup/commit/758e5710282434e30ba61ed9efeea070d94407c1),
+  [3a1cc60](https://github.com/cve-search/vulnerability-lookup/commit/3a1cc604ca1a2ba642ead2b68f9e36ee6a641326))
+- Simplified search page
+  ([f2c55bc](https://github.com/cve-search/vulnerability-lookup/commit/f2c55bc9f7e53caca4d011e06678989c490e9ba2))
+- Improved display of tables and lists generated from Markdown (in comments and bundles)
+  ([24fa4f9](https://github.com/cve-search/vulnerability-lookup/commit/24fa4f95ed4e0266424e71d5d985ae693f22e9e1),
+  [15fe9b2](https://github.com/cve-search/vulnerability-lookup/commit/15fe9b2c673ecf1ab14065e4925869d061dccebe))
+- The ranking of the users is now taking into account the contributions of comments and bundles.
+  Users who have never contributed are sorted by *last_seen*, after the processed result.
+  ([4e4a436](https://github.com/cve-search/vulnerability-lookup/commit/4e4a436ea6052bb23b7bad373c010bf161fff05e))
+- Various graphical and accessibility improvements.
+
+### Fixes
+
+- Do not iterate over meta tags when never defined in an object
+  ([93f9966](https://github.com/cve-search/vulnerability-lookup/commit/93f9966ad4009d8bbe8b54b7efb87a19886e3605))
+
+
 ## 1.5.0 (2024-08-30)
 
 ### News

bin/update.py

@@ -90,9 +90,11 @@ def main() -> None:
     keep_going(args.yes)
     run_command(f'poetry run {(Path("tools") / "validate_config_files.py").as_posix()} --update')
 
-    print('* Migrate database.')
-    keep_going(args.yes)
-    run_command('poetry run flask --app website.app db upgrade')
+    if get_config('generic', 'user_accounts'):
+        print('* Migrate database.')
+        keep_going(args.yes)
+        run_command('poetry run flask --app website.app db_backup')
+        run_command('poetry run flask --app website.app db upgrade')
 
     print('* Restarting')
     keep_going(args.yes)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "vulnerabilitylookup"
-version = "1.5.0"
+version = "1.6.0"
 description = "Vulnerability Lookup facilitates quick correlation of vulnerabilities from various sources, independent of vulnerability IDs, and streamlines the management of Coordinated Vulnerability Disclosure (CVD)."
 authors = ["Alexandre Dulaunoy <alexandre.dulaunoy@circl.lu>", "Raphaël Vinot <raphael.vinot@circl.lu>", "Cédric Bonhomme <cedric.bonhomme@circl.lu>"]
 license = "AGPL-3.0-or-later"
@@ -69,6 +69,7 @@ the-big-username-blacklist = "^1.5.4"
 cvss = "^3.1"
 xmltodict = "^0.13.0"
 feedparser = "^6.0.11"
+pycountry = "^24.6.1"
 
 [tool.poetry.group.dev.dependencies]
 ipython = "^8.26.0"

website/app.py

@@ -30,6 +30,7 @@
 # ##### Registering commands
 
 application.cli.add_command(commands.db_init)
+application.cli.add_command(commands.db_backup)
 application.cli.add_command(commands.create_admin)
 application.cli.add_command(commands.create_user)
 application.cli.add_command(commands.user_list)

website/lib/utils.py

@@ -1,6 +1,15 @@
+from typing import Any
 from typing import List
 
 import re
+import requests
+
+from sqlalchemy import func, desc
+
+from website.models import Bundle
+from website.models import Comment
+from website.models import User
+from website.web.bootstrap import db
 
 
 def find_cve_ids(text: str) -> List[str]:
@@ -43,3 +52,41 @@ def find_pysec_ids(text: str) -> List[str]:
     pysec_ids = list(set(pysec_ids))
 
     return pysec_ids
+
+
+def query_country_code_mmdb(ip_address: str) -> str:
+    """Query the MMDB server from CIRCL in order to get the Aplha-3 code from an IP address."""
+    url = f"https://ip.circl.lu/geolookup/{ip_address}"
+    try:
+        response = requests.get(url)
+    except Exception:
+        return ""
+    if response.status_code == 200:
+        try:
+            return response.json()[0]["country_info"]["Alpha-3 code"]
+        except Exception:
+            return ""
+    else:
+        return ""
+
+
+def top_contributors(limit: int = 3) -> List[Any]:
+    """Return the list of the top 3 (by default) contributors (creation of comments and bundles)."""
+    top_contributors = (
+        db.session.query(
+            User.id,
+            User.login,
+            (func.count(Comment.uuid) + func.count(Bundle.uuid)).label(
+                "total_contributions"
+            ),
+        )
+        .outerjoin(Comment, User.id == Comment.author_id)  # Join User with Comment
+        .outerjoin(Bundle, User.id == Bundle.author_id)  # Join User with Bundle
+        .group_by(User.id)  # Group by User ID to aggregate counts
+        .order_by(desc("total_contributions"))  # Order by total contributions
+    )
+    if limit != -1:
+        top_contributors = top_contributors.limit(
+            limit
+        )  # Limit to the top 3 (by default) contributors
+    return top_contributors.all()

website/migrations/versions/756de632f85b_user_profile.py

@@ -0,0 +1,81 @@
+"""user profile
+
+Revision ID: 756de632f85b
+Revises: 7e42683b12cd
+Create Date: 2024-09-11 08:54:31.814650
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "756de632f85b"
+down_revision = "7e42683b12cd"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("user", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "country_code",
+                sa.String(length=3),
+                server_default=sa.text("''"),
+                nullable=True,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "bio",
+                sa.String(length=5000),
+                server_default=sa.text("''"),
+                nullable=True,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "webpage",
+                sa.String(length=2048),
+                server_default=sa.text("''"),
+                nullable=True,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "mastodon",
+                sa.String(length=500),
+                server_default=sa.text("''"),
+                nullable=True,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "github",
+                sa.String(length=39),
+                server_default=sa.text("''"),
+                nullable=True,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "linkedin",
+                sa.String(length=30),
+                server_default=sa.text("''"),
+                nullable=True,
+            )
+        )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("user", schema=None) as batch_op:
+        batch_op.drop_column("linkedin")
+        batch_op.drop_column("github")
+        batch_op.drop_column("mastodon")
+        batch_op.drop_column("webpage")
+        batch_op.drop_column("bio")
+        batch_op.drop_column("country_code")

website/models/__init__.py

@@ -3,9 +3,10 @@
 
 from sqlalchemy import create_engine, text
 
-from .user import User
 from .bundle import Bundle
 from .comment import Comment
+from .user import User
+
 
 __all__ = ["User", "Bundle", "Comment"]
 

website/models/user.py

@@ -2,6 +2,7 @@
 
 import re
 import secrets
+from sqlalchemy import func
 from urllib.parse import urlparse
 from datetime import datetime
 
@@ -14,6 +15,8 @@
 from validate_email import validate_email  # type: ignore[import-untyped]
 from werkzeug.security import check_password_hash
 
+from website.models import Bundle
+from website.models import Comment
 from website.web.bootstrap import db
 from website.web.bootstrap import application
 
@@ -30,15 +33,24 @@ class User(db.Model, UserMixin):  # type: ignore[name-defined, misc]
     id = db.Column(db.Integer, primary_key=True)
     login = db.Column(db.String(30), unique=True, nullable=False)
     name = db.Column(db.String(50), nullable=False)
-    organisation = db.Column(db.String(50), default="")
+    email = db.Column(db.String(256), nullable=False)
     pwdhash = db.Column(db.String(), nullable=False)
-    secret_token = db.Column(db.String(), unique=True)
+    secret_token = db.Column(db.String(), unique=True)  # for Two-Factor Authentication
     is_two_factor_authentication_enabled = db.Column(
         db.Boolean, nullable=False, default=False
     )
-    email = db.Column(db.String(256), nullable=False)
+
+    organisation = db.Column(db.String(50), default="")
+    country_code = db.Column(db.String(3), default="")  # for alpha-3 codes
+    bio = db.Column(db.String(5000), default="")
+    webpage = db.Column(db.String(2048), default="")
+    mastodon = db.Column(db.String(500), default="")
+    github = db.Column(db.String(39), default="")
+    linkedin = db.Column(db.String(30), default="")
+
     created_at = db.Column(db.DateTime(), default=datetime.now)
     last_seen = db.Column(db.DateTime(), default=datetime.now)
+
     apikey = db.Column(db.String(100), default=generate_token, unique=True)
 
     is_active = db.Column(db.Boolean(), default=True)
@@ -94,16 +106,53 @@ def validates_email(self, key: str, value: str) -> str:
 
     @validates("name")
     def validates_name(self, key: str, value: str) -> str:
-        assert 3 <= len(value) <= 256, AssertionError("Maximum length for name: 256")
+        assert 3 <= len(value) <= 50, AssertionError("Maximum length for name: 50")
+        value = value.strip()
+        return value
+
+    @validates("bio")
+    def validates_bio(self, key: str, value: str) -> str:
+        assert 0 <= len(value) <= 500, AssertionError("Maximum length for bio: 500")
+        value = value.strip()
+        # Remove external links
+        value = re.sub(r"\[.*?\]\(http[s]?://[^\)]+\)", "", value)
+        # Remove images
+        value = re.sub(r"!\[.*?\]\(.*?\)", "", value)
+        return value
+
+    @validates("country_code")
+    def validates_country_code(self, key: str, value: str) -> str:
+        assert 0 <= len(value) <= 3, AssertionError(
+            "Maximum length for country_code: 3"
+        )
         value = value.strip()
         return value
 
     @validates("organisation")
     def validates_organisation(self, key: str, value: str) -> str:
-        assert 3 <= len(value) <= 256, AssertionError(
-            "Maximum length for organisation: 256"
+        assert 0 <= len(value) <= 50, AssertionError(
+            "Maximum length for organisation: 50"
         )
-        value = value.strip()
+        return re.sub("[^a-zA-Z0-9_-]", "", value.strip())
+
+    @validates("github")
+    def validates_github(self, key: str, value: str) -> str:
+        assert 0 <= len(value) <= 39, AssertionError("Maximum length for GitHub: 39")
+        if value.strip():
+            github_regex = r"^[a-zA-Z\d](?:[a-zA-Z\d]|-(?=[a-zA-Z\d])){0,38}$"
+            assert re.match(github_regex, value) is not None, AssertionError(
+                "Invalid GitHub username."
+            )
+        return value
+
+    @validates("linkedin")
+    def validates_linkedin(self, key: str, value: str) -> str:
+        assert 0 <= len(value) <= 30, AssertionError("Maximum length for LinkedIn: 30")
+        if value.strip():
+            linkedin_regex = r"^[a-zA-Z\d](?:[a-zA-Z\d-]{0,28}[a-zA-Z\d])?$"
+            assert re.match(linkedin_regex, value) is not None, AssertionError(
+                "Invalid LinkedIn username."
+            )
         return value
 
     @validates("apikey")
@@ -129,3 +178,20 @@ def is_otp_valid(self, user_otp: Any) -> bool:
         """Checks the validity of a One Time password."""
         totp: TOTP = pyotp.parse_uri(self.get_authentication_setup_uri())  # type: ignore
         return totp.verify(user_otp)
+
+    def nb_contributions(self) -> int:
+        """Returns the number of contributions (creation of comments and bundles) of the user."""
+        user_contributions = (
+            db.session.query(
+                User.id,
+                (func.count(Comment.uuid) + func.count(Bundle.uuid)).label(
+                    "total_contributions"
+                ),
+            )
+            .outerjoin(Comment, User.id == Comment.author_id)  # Join User with Comment
+            .outerjoin(Bundle, User.id == Bundle.author_id)  # Join User with Bundle
+            .filter(User.id == self.id)  # Filter by specific user ID
+            .group_by(User.id)  # Group by User ID
+            .first()
+        )
+        return user_contributions[1]

website/web/api/v1/bundle.py

@@ -214,6 +214,7 @@ def post(self) -> Tuple[ResultType, int]:
         bundle.get("related_vulnerabilities", []).extend(
             find_pysec_ids(bundle.get("description", ""))
         )
+        bundle["related_vulnerabilities"] = list(set(bundle["related_vulnerabilities"]))
 
         try:
             validate_json(bundle, "circl_bundle")

website/web/api/v1/comment.py

@@ -6,7 +6,6 @@
 
 import logging
 import uuid
-from flask import jsonify
 from flask_login import current_user  # type: ignore[import-untyped]
 from flask_restx import fields  # type: ignore[import-untyped]
 from flask_restx import abort
@@ -227,6 +226,9 @@ def post(self) -> Tuple[ResultType, int]:
         comment.get("related_vulnerabilities", []).extend(
             find_pysec_ids(comment.get("description", ""))
         )
+        comment["related_vulnerabilities"] = list(
+            set(comment["related_vulnerabilities"])
+        )
 
         # Validate the JSON payload
         try: