(DOCSP-30602): Ingest service prod pipeline (#42)

* stub out prod pipeline + add monorepo support * more monorepo conf * Release 0.0.2 * add release-it conf for relevant packages * add release-it conf for relevant packages * Release 0.0.3 * Release 0.0.4-0 * Release 0.0.4 * update package * Release 0.0.5 * fix GH actions/pr template * remove unused lerna scripts * trigger rerun * fix package.json * prod env setup * remove old comment * update server release conf * Release 1.0.1 * update package-lock * update release scripts * Release 1.0.2 * remove trigger file * ingest prod pipeline * post rebase cleanup * Release 0.0.6-0 * fix build issues * update lock * fix dependencies
mongodb · Jul 26, 2023 · 8e60755 · 8e60755
1 parent ab166a7
commit 8e60755
Show file tree

Hide file tree

Showing 5 changed files with 3,692 additions and 6,865 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -185,8 +185,9 @@ steps:
       kubernetes_token:
         from_secret: prod_kubernetes_token
 
+# ---
 # Ingest service
-
+# ---
 ---
 depends_on: ["test-all"]
 kind: pipeline
@@ -257,3 +258,72 @@ steps:
       api_server: https://api.staging.corp.mongodb.com
       kubernetes_token:
         from_secret: staging_kubernetes_token
+
+---
+depends_on: ["test-all"]
+kind: pipeline
+type: kubernetes
+name: production-build-ingest-service
+
+trigger:
+  event:
+    - tag
+  ref:
+    include:
+      - refs/tags/ingest-v*
+
+steps:
+  # Builds and publishes Docker image for production
+  - name: publish-production-ingest-service
+    image: plugins/kaniko-ecr
+    settings:
+      dockerfile: ingest-service.dockerfile
+      create_repository: true
+      registry: 795250896452.dkr.ecr.us-east-1.amazonaws.com
+      repo: docs/${DRONE_REPO_NAME}-ingest-service
+      tags:
+        - git-${DRONE_COMMIT_SHA:0:7}-production
+        - latest-production
+      access_key:
+        from_secret: ecr_access_key
+      secret_key:
+        from_secret: ecr_secret_key
+
+  # Promotes current drone build to production environment
+  - name: promote-production-ingest-service
+    image: drone/cli:1.4.0-alpine
+    commands:
+      - drone build promote mongodb/docs-chatbot ${DRONE_BUILD_NUMBER} production
+    environment:
+      DRONE_SERVER: ${DRONE_SYSTEM_PROTO}://${DRONE_SYSTEM_HOST}
+      DRONE_TOKEN:
+        from_secret: drone_token
+
+---
+kind: pipeline
+type: kubernetes
+name: production-deploy-ingest-service
+
+trigger:
+  event:
+    - promote
+  target:
+    - production
+
+steps:
+  # Deploys Docker image associated with production build that triggered promotion
+  # TODO: update to use cron chart
+  - name: deploy-production-ingest-service
+    image: quay.io/mongodb/drone-helm:v3
+    settings:
+      chart: mongodb/cronjobs
+      chart_version: 1.7.1
+      add_repos: [mongodb=https://10gen.github.io/helm-charts]
+      namespace: docs
+      release: ingest-service
+      values: image.tag=git-${DRONE_COMMIT_SHA:0:7}-production,image.repository=795250896452.dkr.ecr.us-east-1.amazonaws.com/docs/${DRONE_REPO_NAME}-ingest-service
+      values_files:
+        - ingest/environments/production.yml
+      api_server: https://api.prod.corp.mongodb.com
+      kubernetes_token:
+        from_secret: prod_kubernetes_token
diff --git a/chat-ui/package.json b/chat-ui/package.json
@@ -29,7 +29,6 @@
     "build": "npm run build:static-site:staging",
     "preview:static-site:dev": "npm run build:static-site:dev && vite preview --base ./ --config vite.config.static-site.js",
     "preview:static-site:staging": "npm run build:static-site:staging && vite preview --base ./ --config vite.config.static-site.js",
-    "build": "npm run build:static-site:staging",
     "format": "prettier ./src --write",
     "format:check": "prettier ./src --check",
     "lint": "eslint ./src --ext js,jsx,ts,tsx --report-unused-disable-directives --max-warnings 0",
@@ -64,6 +63,7 @@
     "stream-browserify": "^3.0.0"
   },
   "devDependencies": {
+    "@rollup/plugin-inject": "^5.0.3",
     "@storybook/addon-essentials": "^7.0.20",
     "@storybook/addon-interactions": "^7.0.20",
     "@storybook/addon-links": "^7.0.20",
@@ -82,14 +82,17 @@
     "eslint-plugin-react-hooks": "^4.6.0",
     "eslint-plugin-react-refresh": "^0.3.4",
     "eslint-plugin-storybook": "^0.6.12",
+    "node-stdlib-browser": "^1.2.0",
+    "postcss": "^8.4.27",
     "prop-types": "^15.8.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-refresh": "^0.14.0",
     "release-it": "^15.11.0",
+    "rollup": "^3.26.3",
     "storybook": "^7.0.20",
     "typescript": "^5.0.2",
-    "vite": "^4.3.9",
+    "vite": "^4.4.7",
     "vite-plugin-libcss": "^1.1.0",
     "vite-plugin-node-polyfills": "^0.9.0"
   }

diff --git a/ingest/environments/production.yml b/ingest/environments/production.yml
@@ -0,0 +1,30 @@
+image:
+  repository: node
+  tag: 18
+
+cronJobs:
+  - name: ingest-all-content
+    schedule: "0 6 * * *" # every day at 6am UTC (1AM EST)
+    command: ["node", ".", "all"]
+    env:
+      OPENAI_EMBEDDING_MODEL: text-embedding-ada-002
+      OPENAI_EMBEDDING_MODEL_VERSION: 2023-03-15-preview
+      MONGODB_DATABASE_NAME: docs-chatbot-prod
+      VECTOR_SEARCH_INDEX_NAME: default
+      OPENAI_CHAT_COMPLETION_MODEL_VERSION: 2023-06-01-preview
+    envSecrets:
+      MONGODB_CONNECTION_URI: docs-chatbot-prod
+      OPENAI_ENDPOINT: docs-chatbot-prod
+      OPENAI_API_KEY: docs-chatbot-prod
+      OPENAI_EMBEDDING_DEPLOYMENT: docs-chatbot-prod
+      OPENAI_CHAT_COMPLETION_DEPLOYMENT: docs-chatbot-prod
+      DEVCENTER_CONNECTION_URI: docs-chatbot-prod
+    resources:
+      # guaranteed amount of resources
+      requests:
+        cpu: 100m
+        memory: 2Gi
+      # maximum allowed resources
+      limits:
+        cpu: 500m
+        memory: 5Gi
diff --git a/ingest/package.json b/ingest/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ingest",
-  "version": "0.0.5",
+  "version": "0.0.6-0",
   "description": "Ingests data sources and stores embeddings.",
   "main": "./build/main.js",
   "bin": {