Merge remote-tracking branch 'core'

IDinsight · Sep 10, 2024 · 92ba632 · 92ba632
1 parent fd4882a
commit 92ba632
Show file tree

Hide file tree

Showing 171 changed files with 7,609 additions and 3,779 deletions.
diff --git a/.github/workflows/build-alignscore.yaml b/.github/workflows/build-alignscore.yaml
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -40,3 +40,11 @@ jobs:
             --title="${{ github.ref_name }}" \
             --update-aliases ${{ github.ref_name }} latest
           mike set-default latest --allow-undefined --push
+
+      - name: Add CNAME
+        run: |
+          git switch gh-pages
+          echo "docs.ask-a-question.com" >> CNAME
+          git add CNAME
+          git commit -m "Add CNAME"
+          git push -f origin gh-pages
diff --git a/.github/workflows/test_speech_api.yaml b/.github/workflows/test_speech_api.yaml
@@ -0,0 +1,50 @@
+name: Speech Unit Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "**"
+    paths:
+      - "optional_components/speech_api/**"
+
+env:
+  PREFERRED_MODEL: base
+  WHISPER_MODEL_DIR: whisper_models
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ffmpeg
+          sudo apt-get clean
+          sudo rm -rf /var/lib/apt/lists/*
+
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r optional_components/speech_api/requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu
+          python -m pip install -r requirements-dev.txt
+          python -m pip install pytest
+
+      - name: Run Speech Unit Tests
+        env:
+          PREFERRED_MODEL: ${{ env.PREFERRED_MODEL }}
+          WHISPER_MODEL_DIR: ${{ env.WHISPER_MODEL_DIR }}
+        run: |
+          mkdir -p $WHISPER_MODEL_DIR
+          cd optional_components/speech_api
+          python -m pytest
+          cd ../..
+          rm -rf $WHISPER_MODEL_DIR
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -11,7 +11,6 @@ env:
   POSTGRES_PASSWORD: postgres-test-pw
   POSTGRES_USER: postgres-test-user
   POSTGRES_DB: postgres-test-db
-  ALIGN_SCORE_API: http://alignscore:5001/alignscore_base
   REDIS_HOST: redis://redis:6379
 jobs:
   container-job:
@@ -40,15 +39,6 @@ jobs:
           --health-retries 5
         ports:
           - 6379:6379
-      alignscore:
-        image: public.ecr.aws/j3r7b4k0/alignscore-base:latest
-        ports:
-          - 5001:5001
-        options: >-
-          --health-cmd "curl -f http://localhost:5001/healthcheck"
-          --health-interval 5s
-          --health-timeout 5s
-          --health-retries 5
     steps:
       - name: install dependencies
         run: apt-get update && apt-get install -y lsb-release && apt-get clean all

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -16,7 +16,7 @@ repos:
     hooks:
       - id: interrogate
         name: interrogate
-        exclude: ^(core_backend/migrations|core_backend/tests|core_backend/validation)
+        exclude: ^(core_backend/migrations|core_backend/tests|core_backend/validation|optional_components/speech_api/tests)
         entry: interrogate
         args: [-v, --fail-under=95, --ignore-module]
   - repo: https://github.com/astral-sh/ruff-pre-commit
@@ -29,14 +29,14 @@ repos:
     hooks:
       - id: mypy
         exclude: ^data/|^scripts/
-        additional_dependencies:
-          [types-PyYAML==6.0.12.12, types-python-dateutil, redis]
+        additional_dependencies: [types-PyYAML==6.0.12.12, types-python-dateutil, redis]
         args: [--ignore-missing-imports, --explicit-package-base]
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: "v3.0.3" # Use the sha / tag you want to point at
     hooks:
       - id: prettier
         exclude: ^(docs)
+        args: ["--print-width", "88"]
   - repo: https://github.com/antonbabenko/pre-commit-terraform
     rev: v1.92.0
     hooks:

diff --git a/.secrets.baseline b/.secrets.baseline
@@ -186,15 +186,6 @@
         "line_number": 63
       }
     ],
-    "core_backend/app/question_answer/schemas.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "core_backend/app/question_answer/schemas.py",
-        "hashed_secret": "5b8b7a620e54e681c584f5b5c89152773c10c253",
-        "is_verified": false,
-        "line_number": 67
-      }
-    ],
     "core_backend/migrations/versions/2023_09_16_c5a948963236_create_query_table.py": [
       {
         "type": "Hex High Entropy String",
@@ -318,49 +309,49 @@
         "filename": "core_backend/tests/api/conftest.py",
         "hashed_secret": "407c6798fe20fd5d75de4a233c156cc0fce510e3",
         "is_verified": false,
-        "line_number": 45
+        "line_number": 44
       },
       {
         "type": "Secret Keyword",
         "filename": "core_backend/tests/api/conftest.py",
         "hashed_secret": "42553e798bc193bcf25368b5e53ec7cd771483a7",
         "is_verified": false,
-        "line_number": 46
+        "line_number": 45
       },
       {
         "type": "Secret Keyword",
         "filename": "core_backend/tests/api/conftest.py",
         "hashed_secret": "9fb7fe1217aed442b04c0f5e43b5d5a7d3287097",
         "is_verified": false,
-        "line_number": 48
+        "line_number": 47
       },
       {
         "type": "Secret Keyword",
         "filename": "core_backend/tests/api/conftest.py",
         "hashed_secret": "767ef7376d44bb6e52b390ddcd12c1cb1b3902a4",
         "is_verified": false,
-        "line_number": 49
+        "line_number": 48
       },
       {
         "type": "Secret Keyword",
         "filename": "core_backend/tests/api/conftest.py",
         "hashed_secret": "70240b5d0947cc97447de496284791c12b2e678a",
         "is_verified": false,
-        "line_number": 54
+        "line_number": 53
       },
       {
         "type": "Secret Keyword",
         "filename": "core_backend/tests/api/conftest.py",
         "hashed_secret": "80fea3e25cb7e28550d13af9dfda7a9bd08c1a78",
         "is_verified": false,
-        "line_number": 55
+        "line_number": 54
       },
       {
         "type": "Secret Keyword",
         "filename": "core_backend/tests/api/conftest.py",
         "hashed_secret": "3465834d516797458465ae4ed2c62e7020032c4e",
         "is_verified": false,
-        "line_number": 312
+        "line_number": 311
       }
     ],
     "core_backend/tests/api/test.env": [
@@ -372,20 +363,29 @@
         "line_number": 5
       }
     ],
-    "core_backend/tests/api/test_dashboard.py": [
+    "core_backend/tests/api/test_dashboard_overview.py": [
       {
         "type": "Secret Keyword",
-        "filename": "core_backend/tests/api/test_dashboard.py",
+        "filename": "core_backend/tests/api/test_dashboard_overview.py",
         "hashed_secret": "233243ef95e736679cb1d5664a4c71ba89c10664",
         "is_verified": false,
         "line_number": 154
       },
       {
         "type": "Secret Keyword",
-        "filename": "core_backend/tests/api/test_dashboard.py",
+        "filename": "core_backend/tests/api/test_dashboard_overview.py",
         "hashed_secret": "6367c48dd193d56ea7b0baad25b19455e529f5ee",
         "is_verified": false,
-        "line_number": 291
+        "line_number": 290
+      }
+    ],
+    "core_backend/tests/api/test_dashboard_performance.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "core_backend/tests/api/test_dashboard_performance.py",
+        "hashed_secret": "1a421e4919b1674defaf1ea063893fe198fe5dd8",
+        "is_verified": false,
+        "line_number": 123
       }
     ],
     "core_backend/tests/api/test_data_api.py": [
@@ -394,7 +394,7 @@
         "filename": "core_backend/tests/api/test_data_api.py",
         "hashed_secret": "233243ef95e736679cb1d5664a4c71ba89c10664",
         "is_verified": false,
-        "line_number": 372
+        "line_number": 367
       }
     ],
     "core_backend/tests/api/test_question_answer.py": [
@@ -410,7 +410,7 @@
         "filename": "core_backend/tests/api/test_question_answer.py",
         "hashed_secret": "6367c48dd193d56ea7b0baad25b19455e529f5ee",
         "is_verified": false,
-        "line_number": 584
+        "line_number": 609
       }
     ],
     "core_backend/tests/api/test_user_tools.py": [
@@ -550,5 +550,5 @@
       }
     ]
   },
-  "generated_at": "2024-08-14T18:24:15Z"
+  "generated_at": "2024-09-10T14:09:26Z"
 }
diff --git a/Makefile b/Makefile
@@ -10,6 +10,8 @@ clean:
 	find . -type f -name "*.py[co]" -delete
 	find . -type d -name "__pycache__" -delete
 	find . -type d -name ".pytest_cache" -exec rm -rf {} +
+	find . -type d -name ".mypy_cache" -exec rm -rf {} +
+	find . -type d -name ".ruff_cache" -exec rm -rf {} +
 
 # Note: Run `make fresh-env psycopg2-binary=true` to manually replace psycopg with psycopg2-binary
 fresh-env :
@@ -32,6 +34,8 @@ setup-dev: setup-db setup-redis add-users-to-db setup-embeddings-arm
 teardown-dev: teardown-db teardown-redis teardown-embeddings
 
 ## Helper targets
+guard-%:
+	@if [ -z '${${*}}' ]; then echo 'ERROR: environment variable $* not set' && exit 1; fi
 
 # Add users to db
 add-users-to-db:
@@ -106,7 +110,7 @@ build-embeddings-arm:
 	@cd ..
 	@rm -rf text-embeddings-inference
 
-setup-embeddings-arm:
+setup-embeddings-arm: guard-HUGGINGFACE_MODEL guard-HUGGINGFACE_EMBEDDINGS_API_KEY
 	-@docker stop huggingface-embeddings
 	-@docker rm huggingface-embeddings
 	@docker system prune -f
@@ -119,7 +123,7 @@ setup-embeddings-arm:
         --model-id $(HUGGINGFACE_MODEL) \
         --api-key $(LITELLM_API_KEY)
 
-setup-embeddings:
+setup-embeddings: guard-HUGGINGFACE_MODEL guard-HUGGINGFACE_EMBEDDINGS_API_KEY
 	-@docker stop huggingface-embeddings
 	-@docker rm huggingface-embeddings
 	@docker system prune -f

diff --git a/admin_app/Dockerfile b/admin_app/Dockerfile
@@ -12,15 +12,15 @@ WORKDIR /app
 COPY --from=deps /app/node_modules ./node_modules
 COPY . .
 
-ENV NEXT_TELEMETRY_DISABLED 1
+ENV NEXT_TELEMETRY_DISABLED=1
 
 RUN npm run build
 
 FROM base AS runner
 WORKDIR /app
 
-ENV NODE_ENV production
-ENV NEXT_TELEMETRY_DISABLED 1
+ENV NODE_ENV=production
+ENV NEXT_TELEMETRY_DISABLED=1
 
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
@@ -36,6 +36,6 @@ USER nextjs
 
 EXPOSE 3000
 
-ENV PORT 3000
+ENV PORT=3000
 
 CMD ["node", "server.js"]