Integration merge branch (#4)

* [coral-web] rename schema fields and update openapi client (cohere-ai#189) * update * update client * Add use agents view env variable. (cohere-ai#192) * Change * Format * [backend] api_key not set for class methods (cohere-ai#191) * fix * easier fix * perf: downgrade rehype-highlight and generate once blob url (cohere-ai#193) * perf: downgrade rehype-highlight and generate once blob url * fix: replace code blocks to iframe in conversation history * refactor: run format * refactor: pr feedback * Add OIDC auth strategy, add Blacklist scaffolding (model/schema/crud) (cohere-ai#190) * add oidc, blacklist logic * Add auth * Add logic for GET /auth * Mods * fix: html preview with citations (cohere-ai#200) * [backend] adding tools for agents, filter tools by agent_id (cohere-ai#204) * changes * lint * filter tools by agent * lint * update tests * lint and squash migrations * fix alembic migration err * lint * merge (#3) * [backend] make deployment field optional in API and DB (cohere-ai#213) --------- Co-authored-by: Tianjing Li <tianjinglimail@gmail.com> Co-authored-by: Tomeu <tomeu@cohere.com> Co-authored-by: Scott <146760070+scott-cohere@users.noreply.github.com> Co-authored-by: Beatrix De Wilde <128378696+BeatrixCohere@users.noreply.github.com> * pulling from main * cleaned backend docker image push * syncing with main (conflicts) --------- Co-authored-by: Scott <146760070+scott-cohere@users.noreply.github.com> Co-authored-by: Beatrix De Wilde <128378696+BeatrixCohere@users.noreply.github.com> Co-authored-by: Tomeu <tomeu@cohere.com> Co-authored-by: Tianjing Li <tianjinglimail@gmail.com>
MiniMap-ai · Jun 17, 2024 · 8ff1b42 · 8ff1b42
1 parent 3cd9a44
commit 8ff1b42
Show file tree

Hide file tree

Showing 5 changed files with 110 additions and 41 deletions.
diff --git a/.github/workflows/deploy_backend.yml b/.github/workflows/deploy_backend.yml
@@ -0,0 +1,53 @@
+name: Deploy Backend
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      tags:
+        description: 'Branch or tag (e.g: v0.0.1)'
+        required: false
+        type: string
+
+env:
+  IMAGE_NAME: cohere-toolkit-backend
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    # needs: build-and-push-backend-image
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Auth
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+
+      - name: Configure Docker
+        run: gcloud auth configure-docker
+
+      # Deploy the image to Cloud Run
+      - name: Deploy to Cloud Run
+        run: |
+          gcloud run deploy ${{ env.IMAGE_NAME }} \
+            --project=${{ vars.GCP_PROJECT_ID }} \
+            --image=${{ vars.REGISTRY }}/${{ env.IMAGE_NAME }} \
+            --platform=managed \
+            --region=us-central1 \
+            --allow-unauthenticated \
+            --update-env-vars=DATABASE_URL=${{ secrets.DATABASE_URL }},COHERE_API_KEY=${{ secrets.COHERE_API_KEY }} \
+            --memory=4Gi \
+            --cpu=4 \
+            --max-instances=1
diff --git a/.github/workflows/docker_push_backend.yml b/.github/workflows/docker_push_backend.yml
@@ -11,37 +11,51 @@ on:
         type: string
 
 env:
-  REGISTRY: ghcr.io
   IMAGE_NAME: cohere-toolkit-backend
 
 jobs:
   build-and-push-backend-image:
     runs-on: ubuntu-latest
-    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
     permissions:
       contents: read
-      packages: write
+
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+
+      - name: Login
+        uses: google-github-actions/setup-gcloud@v0
         with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          project_id: ${{ vars.GCP_PROJECT_ID }}
+          service_account_email: ${{ secrets.SA_EMAIL }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Configure Docker
+        run: gcloud auth configure-docker --quiet
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - run: 'echo "$KEY" > ./key.json'
+        shell: bash
+        env:
+          KEY: ${{ secrets.GCP_SA_KEY }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
-        with:
-          images: ${{ env.REGISTRY }}/cohere-ai/${{ env.IMAGE_NAME }}
-      - name: Build and push Docker image
-        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        uses: docker/metadata-action@v3
         with:
-          context: .
-          tags: ${{ steps.meta.outputs.tags }}
-          file: ./src/backend/Dockerfile
-          push: true
+          images: ${{ vars.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build Docker image
+        run: |
+          docker build -t ${{ env.IMAGE_NAME }} -f ./src/backend/Dockerfile .
+
+      - name: Tag Docker image
+        run: |
+          docker tag ${{ env.IMAGE_NAME }} ${{ steps.meta.outputs.tags }}
 
+      - name: Push Docker image
+        env:
+          THE_TAGS: ${{ steps.meta.outputs.tags }}
+        run: |
+          docker push ${{ steps.meta.outputs.tags }}
 
diff --git a/.github/workflows/docker_push_frontend.yml b/.github/workflows/docker_push_frontend.yml
@@ -11,33 +11,35 @@ on:
         type: string
 
 env:
-  REGISTRY: ghcr.io
   IMAGE_NAME: cohere-toolkit-frontend
 
 jobs:
   build-and-push-frontend-image:
     runs-on: ubuntu-latest
-    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
     permissions:
       contents: read
-      packages: write
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
         with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          project_id: ${{ env.GCP_PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Authenticate Docker to Artifact Registry
+        run: gcloud auth configure-docker ${{ env.REGISTRY }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        uses: docker/metadata-action@v3
         with:
-          images: ${{ env.REGISTRY }}/cohere-ai/${{ env.IMAGE_NAME }}
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
       - name: Build and push Docker image
-        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        uses: docker/build-push-action@v2
         with:
           context: ./src/interfaces/coral_web
           tags: ${{ steps.meta.outputs.tags }}

diff --git a/cloudbuild.yaml b/cloudbuild.yaml
@@ -1,8 +1,8 @@
 substitutions:
-  _GCP_PROJECT_ID: <GCP_PROJECT_ID>
-  _GCP_REGION: <GCP_REGION>
-  _DATABASE_URL: <DB_CONNECTION_STRING>
-  _COHERE_API_KEY: <COHERE_API_KEY>
+  _GCP_PROJECT_ID: $PROJECT_ID
+  _GCP_REGION: $REGION
+  _DATABASE_URL: $DATABASE_URL
+  _COHERE_API_KEY: $COHERE_API_KEY
 
 steps:
 # 1. BACKEND STEPS
@@ -15,8 +15,8 @@ steps:
 # c) Deploy container image to Cloud Run
 - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
   entrypoint: gcloud
-  args: ['run', 'deploy', 'toolkit-backend', 
-  '--image', 'gcr.io/$_GCP_PROJECT_ID/toolkit-backend', 
+  args: ['run', 'deploy', 'toolkit-backend',
+  '--image', 'gcr.io/$_GCP_PROJECT_ID/toolkit-backend',
   '--region', '$_GCP_REGION',
   '--port', '8000',
   '--set-env-vars', 'DATABASE_URL=$_DATABASE_URL',
@@ -33,10 +33,10 @@ steps:
 # e) Apply backend Alembic migrations
 - name: gcr.io/cloud-builders/curl
   entrypoint: bash
-  args: 
+  args:
     - '-c'
     - 'curl -X POST "$(cat /workspace/backend_url.txt)/migrate"'
-  
+
 # 2. FRONTEND STEPS
 # a) Build frontend container image
 - name: 'gcr.io/cloud-builders/docker'
@@ -50,7 +50,7 @@ steps:
   args:
     - '-c'
     - |
-        gcloud run deploy toolkit-frontend --port 4000 --memory 2Gi --image gcr.io/$_GCP_PROJECT_ID/toolkit-frontend --region $_GCP_REGION  --allow-unauthenticated --set-env-vars NEXT_PUBLIC_API_HOSTNAME=$(cat /workspace/backend_url.txt) 
+        gcloud run deploy toolkit-frontend --port 4000 --memory 2Gi --image gcr.io/$_GCP_PROJECT_ID/toolkit-frontend --region $_GCP_REGION  --allow-unauthenticated --set-env-vars NEXT_PUBLIC_API_HOSTNAME=$(cat /workspace/backend_url.txt)
 images:
 - gcr.io/$_GCP_PROJECT_ID/toolkit-backend
 - gcr.io/$_GCP_PROJECT_ID/toolkit-frontend
diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile
@@ -28,7 +28,7 @@ ARG INSTALL_COMMUNITY_DEPS
 COPY pyproject.toml poetry.lock ./
 
 # Install poetry
-RUN pip install --no-cache-dir poetry==1.6.1
+RUN pip install --no-cache-dir --upgrade pip setuptools wheel poetry
 
 # Conditional installation of dependencies
 RUN if [ "$INSTALL_COMMUNITY_DEPS" = "true" ]; then \
@@ -40,7 +40,7 @@ RUN if [ "$INSTALL_COMMUNITY_DEPS" = "true" ]; then \
 COPY src/backend/ src/backend/
 COPY src/community src/community/
 
-# Copy environment variables optionally 
+# Copy environment variables optionally
 # IMPORTANT: Can't be put in the docker-compose, will break tests
 COPY .en[v] .env