Merge pull request #67 from Gurubase/develop

API Support & Slack and Discord Integrations
Gurubase · Feb 3, 2025 · c662df0 · c662df0
2 parents c65ccff + 3c74329
commit c662df0
Show file tree

Hide file tree

Showing 105 changed files with 5,312 additions and 846 deletions.
diff --git a/.github/workflows/frontend-docker-image-dockerhub.yml b/.github/workflows/frontend-docker-image-dockerhub.yml
@@ -37,6 +37,8 @@ jobs:
         NEXT_PUBLIC_SENTRY_ORG=${{ secrets.NEXT_PUBLIC_SENTRY_ORG_PROD_FE }}
         NEXT_PUBLIC_SENTRY_PROJECT=${{ secrets.NEXT_PUBLIC_SENTRY_PROJECT_PROD_FE }}
         NEXT_TELEMETRY_DEBUG=${{ secrets.NEXT_TELEMETRY_DEBUG_PROD_FE }}
+        NEXT_PUBLIC_SLACK_INTEGRATION_URL=${{ secrets.NEXT_PUBLIC_SLACK_INTEGRATION_URL_PROD_FE }}
+        NEXT_PUBLIC_DISCORD_INTEGRATION_URL=${{ secrets.NEXT_PUBLIC_DISCORD_INTEGRATION_URL_PROD_FE }}
         AUTH0_SECRET=${{ secrets.AUTH0_SECRET_PROD_FE }}
         AUTH0_BASE_URL=${{ secrets.AUTH0_BASE_URL_PROD_FE }}
         AUTH0_ISSUER_BASE_URL=${{ secrets.AUTH0_ISSUER_BASE_URL_PROD_FE }}
@@ -58,6 +60,8 @@ jobs:
         NEXT_PUBLIC_BACKEND_FETCH_URL_CLIENT=${{ secrets.NEXT_PUBLIC_BACKEND_FETCH_URL_CLIENT_STAGING_FE }}
         NEXT_PUBLIC_PAGE_VISIT_MODULE=${{ secrets.NEXT_PUBLIC_PAGE_VISIT_MODULE_STAGING_FE }}
         NEXT_TELEMETRY_DEBUG=${{ secrets.NEXT_TELEMETRY_DEBUG_STAGING_FE }}
+        NEXT_PUBLIC_SLACK_INTEGRATION_URL=${{ secrets.NEXT_PUBLIC_SLACK_INTEGRATION_URL_STAGING_FE }}
+        NEXT_PUBLIC_DISCORD_INTEGRATION_URL=${{ secrets.NEXT_PUBLIC_DISCORD_INTEGRATION_URL_STAGING_FE }}
         AUTH0_SECRET=${{ secrets.AUTH0_SECRET_STAGING_FE }}
         AUTH0_BASE_URL=${{ secrets.AUTH0_BASE_URL_STAGING_FE }}
         AUTH0_ISSUER_BASE_URL=${{ secrets.AUTH0_ISSUER_BASE_URL_STAGING_FE }}

diff --git a/.github/workflows/frontend-snyk-scan.yml b/.github/workflows/frontend-snyk-scan.yml
diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml
@@ -0,0 +1,25 @@
+name: Documentation
+
+on:
+  push:
+    branches:
+      - master
+      - develop
+  pull_request:
+    branches:
+      - master
+      - develop
+
+jobs:
+  link-checker:
+    name: Check links
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+
+      - name: Check the links
+        uses: lycheeverse/lychee-action@v1
+        with:
+          args: --max-concurrency 1 -v *.md **/*.md
+          fail: true
diff --git a/.lycheeignore b/.lycheeignore
@@ -0,0 +1,3 @@
+http://localhost:8029/
+https://platform.openai.com/api-keys
+https://mastodon.social/@gurubaseio
diff --git a/src/gurubase-backend/.vscode/launch.json b/src/gurubase-backend/.vscode/launch.json
@@ -74,6 +74,17 @@
                 "django_celery_beat.schedulers:DatabaseScheduler",
             ],
             "justMyCode": false,
+        },
+        {
+            "name": "Discord Listener",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/backend/manage.py",
+            "args": [
+                "discordListener"
+            ],
+            "django": true,
+            "justMyCode": false,
         }
     ]
 }
diff --git a/src/gurubase-backend/.vscode/tasks.json b/src/gurubase-backend/.vscode/tasks.json
@@ -101,223 +101,25 @@
             "problemMatcher": []
         },
         {
-            "label": "Populate alaz requests",
+            "label": "discord_listener",
             "type": "shell",
-            "command": "python manage.py populate_alaz_requests ${input:edges} ${input:req_per_edge} ${input:monitoring_id}",
+            "command": "python manage.py discordListener",
             "options": {
                 "cwd": "${workspaceFolder}/backend"
             },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
-        },
-        {
-            "label": "Populate alaz resources",
-            "type": "shell",
-            "command": "python manage.py populate_mock_resources ${input:count} ${input:monitoring_id}",
-            "options": {
-                "cwd": "${workspaceFolder}/backend"
-            },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
-        },
-        {
-            "label": "Clear alaz requests",
-            "type": "shell",
-            "command": "python manage.py clear_alaz_requests ${input:monitoring_id}",
-            "options": {
-                "cwd": "${workspaceFolder}/backend"
-            },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
-        },
-        {
-            "label": "Create dummy containers",
-            "type": "shell",
-            "command": "python manage.py create_dummy_containers ${input:name} ${input:pod} ${input:cluster_uid} ${input:count}",
-            "options": {
-                "cwd": "${workspaceFolder}/backend"
-            },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
-        },
-        {
-            "label": "Get asked resources",
-            "type": "shell",
-            "command": "python manage.py get_asked_resources ${input:type} ${input:cluster_uid} ${input:update_count} ${input:create_count}",
-            "options": {
-                "cwd": "${workspaceFolder}/backend"
-            },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
-        },
-        {
-            "label": "Generate request payload in line protocol for influxdb",
-            "type": "shell",
-            "command": "python manage.py generate_alaz_request_payload_for_influxdb ${input:start_timestamp_ms} ${input:end_timestamp_ms} ${input:monitoring_id} ${input:file_name}",
-            "options": {
-                "cwd": "${workspaceFolder}/backend"
-            },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
-        },
-        {
-            "label": "Generate dist_tracing resources",
-            "type": "shell",
-            "command": "python manage.py populate_dist_tracing ${input:traffic_count} ${input:span_count} ${input:trace_count} ${input:monitoring_id}",
-            "options": {
-                "cwd": "${workspaceFolder}/backend"
-            },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
-        },
-        {
-            "label": "Generate random users",
-            "type": "shell",
-            "command": "python manage.py generate_random_users ${input:count}",
-            "options": {
-                "cwd": "${workspaceFolder}/backend"
-            },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
-        },
-        {
-            "label": "Generate random clusters",
-            "type": "shell",
-            "command": "python manage.py generate_random_clusters ${input:count}",
-            "options": {
-                "cwd": "${workspaceFolder}/backend"
-            },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
-        },
-        {
-            "label": "Generate cluster created events",
-            "type": "shell",
-            "command": "python manage.py generate_cluster_created ${input:count}",
-            "options": {
-                "cwd": "${workspaceFolder}/backend"
-            },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
-        },
-        {
-            "label": "Generate test created events",
-            "type": "shell",
-            "command": "python manage.py generate_test_created ${input:count}",
-            "options": {
-                "cwd": "${workspaceFolder}/backend"
-            },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
-        },
-        {
-            "label": "Generate user logged in events",
-            "type": "shell",
-            "command": "python manage.py generate_user_logged_in ${input:count}",
-            "options": {
-                "cwd": "${workspaceFolder}/backend"
-            },
-            "problemMatcher": [
-                "$eslint-compact"
-            ]
+            "problemMatcher": []
         }
     ],
     "inputs": [
-        {
-            "id": "count",
-            "type": "promptString",
-            "description": "Number of resources",
-            "default": "50"
-        },
-        {
-            "id": "update_count",
-            "type": "promptString",
-            "description": "Number of resources to update",
-            "default": "50"
-        },
-        {
-            "id": "create_count",
-            "type": "promptString",
-            "description": "Number of resources to create",
-            "default": "50"
-        },
-        {
-            "id": "edges",
-            "type": "promptString",
-            "description": "Number of edges to populate",
-            "default": "50"
-        },
-        {
-            "id": "req_per_edge",
-            "type": "promptString",
-            "description": "Number of requests per edge",
-            "default": "20"
-        },
-        {
-            "id": "monitoring_id",
-            "type": "promptString",
-            "description": "Monitoring id"
-        },
-        {
-            "id": "cluster_uid",
-            "type": "promptString",
-            "description": "Cluster uid"
-        },
         {
             "id": "name",
             "type": "promptString",
             "description": "Name of the resource",
         },
-        {
-            "id": "pod",
-            "type": "promptString",
-            "description": "Pod uid",
-        },
         {
             "id": "type",
             "type": "promptString",
             "description": "Type of the resource",
         },
-        {
-            "id": "start_timestamp_ms",
-            "type": "promptString",
-            "description": "Start timestamp in milliseconds",
-        },
-        {
-            "id": "end_timestamp_ms",
-            "type": "promptString",
-            "description": "End timestamp in milliseconds",
-        },
-        {
-            "id": "file_name",
-            "type": "promptString",
-            "description": "File name",
-        },
-        {
-            "id": "traffic_count",
-            "type": "promptString",
-            "description": "Traffic count",
-        },
-        {
-            "id": "span_count",
-            "type": "promptString",
-            "description": "Span count",
-        },
-        {
-            "id": "trace_count",
-            "type": "promptString",
-            "description": "Trace count",
-        }
     ]
 }
diff --git a/src/gurubase-backend/ARCHITECTURE.md b/src/gurubase-backend/ARCHITECTURE.md
@@ -6,7 +6,7 @@ Gurubase is a scalable RAG (Retrieval Augmented Generation) system built to prov
 
 ## System Components
 
-![Gurubase Architecture](architecture.png)
+![Gurubase Architecture](imgs/architecture.png)
 
 ### Frontend (Next.js)
 - Built with Next.js
@@ -195,7 +195,7 @@ The task is run in three stages:
 
 An example of the summarization process looks like this:
 
-![Data Source Summarization](ds-summarization.png)
+![Data Source Summarization](imgs/ds-summarization.png)
 
 In this example, the red summarizations are the initial summarizations, and the blue summarizations are the recursive summarizations. And summarization 11 is the final summarization of the whole guru type. While the first stage creates summarizations 1-6, the second stage creates summarizations 7 and 8, and the third stage creates summarizations 9-11.
 
@@ -295,7 +295,7 @@ For any questions related to date, remember today's date is {date}.
 
 #### 2. Context Retrieval
 
-![Context Retrieval](context-retrieval.png)
+![Context Retrieval](imgs/context-retrieval.png)
 
 The `vector_db_fetch` function in `backend/core/utils.py` is responsible for context retrieval. It happens in multiple stages:
 

diff --git a/src/gurubase-backend/Dockerfile b/src/gurubase-backend/Dockerfile
@@ -8,6 +8,8 @@ ENV LANG=C.UTF-8
 COPY ./requirements.txt .
 RUN apt update -y && apt install -y gcc libcairo2 libcairo2-dev git && rm -rf /var/lib/apt/lists/*
 RUN pip --no-cache-dir install -r requirements.txt
+RUN playwright install
+RUN playwright install-deps
 
 WORKDIR /workspace
 

diff --git a/src/gurubase-backend/Dockerfile.dev b/src/gurubase-backend/Dockerfile.dev
@@ -12,6 +12,8 @@ RUN apt update -y && \
 
 COPY ./requirements.txt .
 RUN pip --no-cache-dir install -r requirements.txt
+RUN playwright install
+RUN playwright install-deps
 
 WORKDIR /workspace