Merge remote-tracking branch 'upstream/feature/nestbot-assistant' into MVP

Author: Dishant1804

.github/CODEOWNERS

@@ -3,6 +3,4 @@
 # For more information on CODEOWNERS, see:
 # https://help.github.com/en/articles/about-code-owners
 
-* @arkid15r
-/cspell/ @arkid15r @kasya
-/frontend/ @arkid15r @kasya
+* @arkid15r @kasya

.github/workflows/run-ci-cd.yaml

@@ -346,7 +346,8 @@ jobs:
           platforms: linux/amd64
           push: true
           secrets: |
-            SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+            RELEASE_VERSION=${{ needs.set-release-version.outputs.release_version }}
+            SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }}
           tags: owasp/nest:frontend-staging
 
       - name: Get frontend image size
@@ -665,7 +666,8 @@ jobs:
           platforms: linux/amd64
           push: true
           secrets: |
-            SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+            RELEASE_VERSION=${{ needs.set-release-version.outputs.release_version }}
+            SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }}
           tags: owasp/nest:frontend-production
 
       - name: Get frontend image size

.github/workflows/run-code-ql.yaml

@@ -4,9 +4,11 @@ on:
   merge_group:
   pull_request:
     branches:
+      - feature/*
       - main
   push:
     branches:
+      - feature/*
       - main
   workflow_dispatch:
 
@@ -29,7 +31,7 @@ jobs:
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@2d92b76c45b91eb80fc44c74ce3fce0ee94e8f9d
+        uses: github/codeql-action/init@192325c86100d080feab897ff886c34abd4c83a3
         with:
           languages: ${{ matrix.language }}
 
@@ -53,6 +55,6 @@ jobs:
         run: pnpm install --frozen-lockfile
 
       - name: Perform CodeQL analysis
-        uses: github/codeql-action/analyze@2d92b76c45b91eb80fc44c74ce3fce0ee94e8f9d
+        uses: github/codeql-action/analyze@192325c86100d080feab897ff886c34abd4c83a3
         with:
           category: /language:${{ matrix.language }}

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
         exclude: (.github|pnpm-lock.yaml)
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.12.12
+    rev: v0.13.0
     hooks:
       - id: ruff
         args:
@@ -38,7 +38,7 @@ repos:
         exclude: (.github|pnpm-lock.yaml)
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.17.1
+    rev: v1.18.1
     hooks:
       - id: mypy
         additional_dependencies:

CONTRIBUTING.md

@@ -400,7 +400,7 @@ make check
 
 This command runs linters and other static analysis tools for both the frontend and backend.
 
-We utilize third-party tools such as CodeRabbit and SonarQube for code review, static analysis, and quality checks. As a contributor, it's your responsibility to address or mark as resolved all issues and suggestions reported by these tools during your pull request review. If a suggestion is valid, please implement it; if not, you may mark it as resolved with a brief explanation. If you're uncertain about a particular suggestion, feel free to leave a comment optionally tagging project leaders or mentors you're working with for further guidance.
+We utilize third-party tools such as CodeRabbit, GitHub Advanced Security, and SonarQube for code review, static analysis, and quality checks. As a contributor, it's your responsibility to address (mark as resolved) all issues and suggestions reported by these tools during your pull request review. If a suggestion is valid, please implement it; if not, you may mark it as resolved with a brief explanation. If you're uncertain about a particular suggestion, feel free to leave a comment optionally tagging project maintainer(s) you're working with for further guidance.
 
 **Please note that your pull request will not be reviewed until all code quality checks pass and all automated suggestions have been addressed or resolved.**
 

backend/apps/ai/Makefile

@@ -34,6 +34,14 @@ ai-update-project-context:
 	@echo "Updating project context"
 	@CMD="python manage.py ai_update_project_context" $(MAKE) exec-backend-command
 
+ai-update-repository-chunks:
+	@echo "Updating repository chunks"
+	@CMD="python manage.py ai_update_repository_chunks" $(MAKE) exec-backend-command
+
+ai-update-repository-context:
+	@echo "Updating repository context"
+	@CMD="python manage.py ai_update_repository_context" $(MAKE) exec-backend-command
+
 ai-update-slack-message-chunks:
 	@echo "Updating Slack message chunks"
 	@CMD="python manage.py ai_update_slack_message_chunks" $(MAKE) exec-backend-command

backend/apps/ai/common/base/chunk_command.py

@@ -7,6 +7,7 @@
 from apps.ai.common.utils import create_chunks_and_embeddings
 from apps.ai.models.chunk import Chunk
 from apps.ai.models.context import Context
+from apps.common.utils import is_valid_json
 
 
 class BaseChunkCommand(BaseAICommand):
@@ -43,10 +44,14 @@ def process_chunks_batch(self, entities: list[Model]) -> int:
                     count, _ = context.chunks.all().delete()
                     self.stdout.write(f"Deleted {count} stale chunks for {entity_key}")
 
-                prose_content, metadata_content = self.extract_content(entity)
-                full_content = (
-                    f"{metadata_content}\n\n{prose_content}" if metadata_content else prose_content
-                )
+                content, metadata_content = self.extract_content(entity)
+
+                if is_valid_json(content):
+                    full_content = content
+                else:
+                    full_content = (
+                        f"{metadata_content}\n\n{content}" if metadata_content else content
+                    )
 
                 if not full_content.strip():
                     self.stdout.write(f"No content to chunk for {self.entity_name} {entity_key}")

backend/apps/ai/common/constants.py

@@ -4,5 +4,6 @@
 DEFAULT_CHUNKS_RETRIEVAL_LIMIT = 5
 DEFAULT_SIMILARITY_THRESHOLD = 0.4
 DELIMITER = "\n\n"
+GITHUB_REQUEST_INTERVAL_SECONDS = 0.5
 MIN_REQUEST_INTERVAL_SECONDS = 1.2
 QUEUE_RESPONSE_TIME_MINUTES = 1

backend/apps/ai/common/extractors/repository.py

@@ -0,0 +1,176 @@
+"""Content extractor for Repository."""
+
+import json
+import logging
+import time
+
+from apps.ai.common.constants import DELIMITER, GITHUB_REQUEST_INTERVAL_SECONDS
+from apps.common.utils import is_valid_json
+from apps.github.utils import get_repository_file_content
+
+logger = logging.getLogger(__name__)
+
+
+def extract_repository_content(repository) -> tuple[str, str]:
+    """Extract structured content from repository data.
+
+    Args:
+        repository: Repository instance
+
+    Returns:
+        tuple[str, str]: (json_content, metadata_content)
+
+    """
+    repository_data = {}
+
+    if repository.name:
+        repository_data["name"] = repository.name
+    if repository.key:
+        repository_data["key"] = repository.key
+    if repository.description:
+        repository_data["description"] = repository.description
+    if repository.homepage:
+        repository_data["homepage"] = repository.homepage
+    if repository.license:
+        repository_data["license"] = repository.license
+    if repository.topics:
+        repository_data["topics"] = repository.topics
+
+    status = {}
+    if repository.is_archived:
+        status["archived"] = True
+    if repository.is_empty:
+        status["empty"] = True
+    if repository.is_owasp_repository:
+        status["owasp_repository"] = True
+    if repository.is_owasp_site_repository:
+        status["owasp_site_repository"] = True
+    if status:
+        repository_data["status"] = status
+
+    funding = {}
+    if repository.is_funding_policy_compliant:
+        funding["policy_compliant"] = True
+    if repository.has_funding_yml:
+        funding["has_funding_yml"] = True
+    if funding:
+        repository_data["funding"] = funding
+
+    if repository.pages_status:
+        repository_data["pages_status"] = repository.pages_status
+
+    features = []
+    if repository.has_downloads:
+        features.append("downloads")
+    if repository.has_issues:
+        features.append("issues")
+    if repository.has_pages:
+        features.append("pages")
+    if repository.has_projects:
+        features.append("projects")
+    if repository.has_wiki:
+        features.append("wiki")
+    if features:
+        repository_data["features"] = features
+
+    stats = {}
+    if repository.commits_count:
+        stats["commits"] = repository.commits_count
+    if repository.contributors_count:
+        stats["contributors"] = repository.contributors_count
+    if repository.forks_count:
+        stats["forks"] = repository.forks_count
+    if repository.open_issues_count:
+        stats["open_issues"] = repository.open_issues_count
+    if repository.stars_count:
+        stats["stars"] = repository.stars_count
+    if repository.subscribers_count:
+        stats["subscribers"] = repository.subscribers_count
+    if repository.watchers_count:
+        stats["watchers"] = repository.watchers_count
+    if stats:
+        repository_data["statistics"] = stats
+
+    dates = {}
+    if repository.created_at:
+        dates["created"] = repository.created_at.strftime("%Y-%m-%d")
+    if repository.updated_at:
+        dates["last_updated"] = repository.updated_at.strftime("%Y-%m-%d")
+    if repository.pushed_at:
+        dates["last_pushed"] = repository.pushed_at.strftime("%Y-%m-%d")
+    if dates:
+        repository_data["dates"] = dates
+
+    ownership = {}
+    if repository.organization:
+        ownership["organization"] = repository.organization.login
+    if repository.owner:
+        ownership["owner"] = repository.owner.login
+    if ownership:
+        repository_data["ownership"] = ownership
+
+    markdown_files = [
+        "README.md",
+        "index.md",
+        "info.md",
+        "leaders.md",
+    ]
+
+    if repository.organization:
+        owner = repository.organization.login
+    else:
+        owner = repository.owner.login if repository.owner else ""
+    branch = repository.default_branch or "main"
+
+    tab_files = []
+    if owner and repository.key:
+        contents_url = (
+            f"https://api.github.com/repos/{owner}/{repository.key}/contents/?ref={branch}"
+        )
+        response = get_repository_file_content(contents_url)
+        if response and is_valid_json(response):
+            items = json.loads(response)
+            for item in items:
+                name = item.get("name", "")
+                if name.startswith("tab_") and name.endswith(".md"):
+                    tab_files.append(name)
+
+    all_markdown_files = markdown_files + tab_files
+
+    markdown_content = {}
+    for file_path in all_markdown_files:
+        try:
+            if owner and repository.key:
+                raw_url = (
+                    f"https://raw.githubusercontent.com/{owner}/{repository.key}/"
+                    f"{branch}/{file_path}"
+                )
+                content = get_repository_file_content(raw_url)
+
+                if content and content.strip():
+                    markdown_content[file_path] = content
+                time.sleep(GITHUB_REQUEST_INTERVAL_SECONDS)
+
+        except (ValueError, TypeError, OSError):
+            logger.debug("Failed to fetch markdown file")
+            continue
+
+    if markdown_content:
+        repository_data["markdown_content"] = markdown_content
+
+    json_content = json.dumps(repository_data, indent=2)
+
+    metadata_parts = []
+    if repository.name:
+        metadata_parts.append(f"Repository Name: {repository.name}")
+    if repository.key:
+        metadata_parts.append(f"Repository Key: {repository.key}")
+    if repository.organization:
+        metadata_parts.append(f"Organization: {repository.organization.login}")
+    if repository.owner:
+        metadata_parts.append(f"Owner: {repository.owner.login}")
+
+    return (
+        json_content,
+        DELIMITER.join(filter(None, metadata_parts)),
+    )

backend/apps/ai/management/commands/ai_update_repository_chunks.py

@@ -0,0 +1,41 @@
+"""A command to create chunks of OWASP repository data for RAG."""
+
+from django.db.models import QuerySet
+
+from apps.ai.common.base.chunk_command import BaseChunkCommand
+from apps.ai.common.extractors.repository import extract_repository_content
+from apps.github.models.repository import Repository
+
+
+class Command(BaseChunkCommand):
+    key_field_name = "key"
+    model_class = Repository
+
+    def __init__(self, *args, **kwargs):
+        """Initialize command for repository."""
+        super().__init__(*args, **kwargs)
+        self.entity_name_plural = "repositories"
+
+    def extract_content(self, entity: Repository) -> tuple[str, str]:
+        """Extract content from the repository."""
+        return extract_repository_content(entity)
+
+    def get_base_queryset(self) -> QuerySet:
+        """Return the base queryset with filtering for OWASP site repositories."""
+        return (
+            super()
+            .get_base_queryset()
+            .filter(
+                is_owasp_site_repository=True,
+                is_archived=False,
+                is_empty=False,
+            )
+        )
+
+    def get_default_queryset(self) -> QuerySet:
+        """Override to avoid is_active filter since Repository doesn't have that field."""
+        return self.get_base_queryset()
+
+    def source_name(self) -> str:
+        """Return the source name for context creation."""
+        return "owasp_repository"