Clean Up Ocular Docker Commands (OcularEngineering#105)

* Modify Local Ocular Env

* Add New Command npm run local

* Fixed Docker Local Command

* Optimize Docker Commands

* Fix Kafka Command

* Clean Up Docker Commands

.gitignore

@@ -35,4 +35,5 @@ typesense-data/
 redis-data/
 
 env.local
+.env.local
 Kafka 

docker-compose.dev.yml

@@ -7,7 +7,6 @@ services:
       dockerfile: packages/ocular-models-server/Dockerfile.models
     ports:
       - 8000:8000
-
   qdrant:
     image: qdrant/qdrant:latest
     restart: always
@@ -19,9 +18,6 @@ services:
       - 6333
       - 6334
       - 6335
-    configs:
-      - source: qdrant_config
-        target: /qdrant/config/production.yaml
     volumes:
       - ./qdrant_data:/qdrant_data
 
@@ -34,27 +30,32 @@ services:
       - ./redis-data:/data
 
   zookeeper:
-    image: "bitnami/zookeeper:latest"
+    image: "confluentinc/cp-zookeeper:latest"
     ports:
       - "2181:2181"
     environment:
-      - ALLOW_ANONYMOUS_LOGIN=yes
+      - ZOOKEEPER_CLIENT_PORT=2181
+      - ZOOKEEPER_TICK_TIME=2000
+    volumes:
+      - ./log4j.properties:/opt/bitnami/zookeeper/conf/log4j.properties
 
   kafka:
-    image: "bitnami/kafka:latest"
+    image: confluentinc/cp-kafka:latest
     user: root
     ports:
       - "9092:9092"
+    depends_on:
+      - zookeeper
     environment:
       - KAFKA_BROKER_ID=1
       - KAFKA_LISTENERS=PLAINTEXT://:9092
       - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092
       - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
       - ALLOW_PLAINTEXT_LISTENER=yes
+      - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR
     volumes:
       - ./Kafka:/bitnami/kafka
-    depends_on:
-      - zookeeper
+      - ./log4j.properties:/etc/kafka/log4j.properties
 
   ocular-db:
     image: postgres:14-alpine
@@ -75,7 +76,7 @@ services:
     environment:
       - DATABASE_URL=postgresql://ocular:ocular@ocular-db:5432/ocular
       - DATABASE_NAME=ocular
-    command: npm run typeorm migration:run
+    command: npm run typeorm:local migration:run
 
   pgadmin:
     image: dpage/pgadmin4

docker-compose.local.yml

@@ -1,5 +1,12 @@
 version: "3"
 services:
+  ocular-models-server:
+    container_name: ocular-models-server
+    build:
+      context: .
+      dockerfile: packages/ocular-models-server/Dockerfile.models
+    ports:
+      - 8000:8000
   qdrant:
     image: qdrant/qdrant:latest
     restart: always
@@ -11,9 +18,6 @@ services:
       - 6333
       - 6334
       - 6335
-    configs:
-      - source: qdrant_config
-        target: /qdrant/config/production.yaml
     volumes:
       - ./qdrant_data:/qdrant_data
 
@@ -25,27 +29,46 @@ services:
     volumes:
       - ./redis-data:/data
 
+  zookeeper:
+    image: "confluentinc/cp-zookeeper:latest"
+    ports:
+      - "2181:2181"
+    environment:
+      - ZOOKEEPER_CLIENT_PORT=2181
+      - ZOOKEEPER_TICK_TIME=2000
+    volumes:
+      - ./log4j.properties:/opt/bitnami/zookeeper/conf/log4j.properties
+
+  kafka:
+    image: confluentinc/cp-kafka:latest
+    user: root
+    ports:
+      - "9092:9092"
+      - "29092:29092"
+    depends_on:
+      - zookeeper
+    environment:
+      - KAFKA_BROKER_ID=1
+      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - KAFKA_LISTENERS=PLAINTEXT://:9092,EXTERNAL://kafka:29092
+      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://:9092,EXTERNAL://kafka:29092
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
+      - ALLOW_PLAINTEXT_LISTENER=yes
+      - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR
+    volumes:
+      - ./Kafka:/bitnami/kafka
+      - ./log4j.properties:/etc/kafka/log4j.properties
+
   ocular-db:
     image: postgres:14-alpine
     ports:
-      - "5432:5432"
+      - "5433:5432"
     environment:
       POSTGRES_PASSWORD: ocular
       POSTGRES_USER: ocular
       POSTGRES_DB: ocular
 
-  ocular-db-migration:
-    container_name: ocular-db-migration
-    depends_on:
-      - ocular-db
-    build:
-      context: .
-      dockerfile: packages/ocular/Dockerfile.local
-    environment:
-      - DATABASE_URL=postgresql://ocular:ocular@ocular-db:5432/ocular
-      - DATABASE_NAME=ocular
-    command: npm run typeorm migration:run
-
   ocular-backend:
     container_name: ocular-backend
     build:
@@ -54,22 +77,24 @@ services:
     depends_on:
       ocular-db:
         condition: service_started
-      ocular-db-migration:
-        condition: service_completed_successfully
       qdrant:
         condition: service_started
-      # typesense:
-      #   condition: service_started
+      ocular-models-server:
+        condition: service_started
+      kafka:
+        condition: service_started
     ports:
       - 9000:9000
+    command: /bin/sh -c "npm run typeorm:local migration:run && npm run local"
     environment:
       - DATABASE_URL=postgresql://ocular:ocular@ocular-db:5432/ocular
       - DATABASE_NAME=ocular
       - NODE_ENV=development
       - QDRANT_DB_URL=http://qdrant:6333
-      # - TYPESENSE_HOST=typesense
       - UI_CORS=http://ocular-ui:9000
       - REDIS_URL=redis:6379
+      - KAFKA_URL=kafka:29092
+      - OCULAR_MODELS_SERVER_URL=http://ocular-models-server:8000
 
   ocular-ui:
     container_name: ocular-ui
@@ -80,8 +105,3 @@ services:
       - 3001:3001
     depends_on:
       - ocular-backend
-
-configs:
-  qdrant_config:
-    content: |
-      log_level: INFO

env.dev.example → env.dev

@@ -1,27 +1,18 @@
-# Database URL + REDIS + QDRANT running Docker
 NODE_ENV=development
 DATABASE_URL=postgresql://ocular:ocular@localhost:5433/ocular
 DATABASE_NAME=ocular
 QDRANT_DB_URL=http://localhost:6333
 UI_CORS=http://localhost:9000
 REDIS_URL=redis://localhost:6379
 KAFKA_URL=localhost:9092
-
-
-# Required Azure Open AI credentials
-AZURE_OPEN_AI_KEY=
-AZURE_OPEN_AI_ENDPOINT=
-AZURE_OPEN_AI_EMBEDDER_DEPLOYMENT_NAME=
-AZURE_OPEN_AI_EMBEDDING_MODEL=
-AZURE_OPEN_AI_CHAT_DEPLOYMENT_NAME=
-AZURE_OPEN_AI_CHAT_MODEL=
+USE_LOCAL_EMBEDDER=true # Use local embedder instead of Azure or OpenAI
+OCULAR_MODELS_SERVER_URL=http://localhost:8000
 
 # Required Open AI credentials
 OPEN_AI_KEY=
-OPEN_AI_EMBEDDING_MODEL=
 OPEN_AI_CHAT_MODEL=
 
-# Add these if you want to enable index data from apps
+# Add these if you want to use OAuth to index data from apps
 
 # Google (Gmail + Google Drive)
 GOOGLE_CLIENT_ID=

env.local

@@ -0,0 +1,11 @@
+# Required Open AI Keys
+OPEN_AI_KEY=
+OPEN_AI_CHAT_MODEL=
+
+# OAUTH Credientials - Add these if you want to enable OAUTH for indexing apps
+# Apps Installed
+GOOGLE_CLIENT_ID=
+GOOGLE_CLIENT_SECRET=
+
+ASANA_CLIENT_ID=
+ASANA_CLIENT_SECRET=

log4j.properties

@@ -0,0 +1,23 @@
+
+log4j.rootLogger=INFO, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+
+log4j.logger.kafka=ERROR
+log4j.logger.kafka.network.RequestChannel$=WARN
+log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG
+log4j.logger.kafka.request.logger=WARN
+log4j.logger.kafka.controller=ERROR
+log4j.logger.kafka.log.LogCleaner=INFO
+log4j.logger.state.change.logger=ERROR
+log4j.logger.kafka.authorizer.logger=WARN
+log4j.logger.org.apache.zookeeper=ERROR
+log4j.logger.org.apache.kafka=ERROR
+log4j.logger.kafka.cluster=ERROR
+log4j.logger.kafka.coordinator=ERROR
+log4j.logger.kafka.log=ERROR
+log4j.logger.kafka.server=ERROR
+log4j.logger.kafka.zookeeper=ERROR

packages/ocular-models-server/server.py

@@ -17,7 +17,6 @@
 model.eval()
 
 def embed(docs: list[str]) -> list[list[float]]:
-    print(docs)
     # tokenize
     tokens = tokenizer(
         docs, padding=True, max_length=512, truncation=True, return_tensors="pt"

packages/ocular-telemetry/src/telemeter.js

@@ -51,7 +51,7 @@ class Telemeter {
     let enabled = this.store_.getConfig(`telemetry.enabled`)
     if (enabled === undefined || enabled === null) {
       if (!isCI()) {
-        showAnalyticsNotification()
+        // showAnalyticsNotification()
       }
       enabled = true
       this.store_.setConfig(`telemetry.enabled`, enabled)

packages/ocular-telemetry/src/util/show-notification.js

@@ -8,7 +8,7 @@ const defaultConfig = {
 
 const defaultMessage =
   `Ocular collects anonymous usage analytics\n` +
-  `to help improve Medusa for all users.\n` +
+  `to help improve Ocular for all users.\n` +
   `\n` +
   `If you'd like to opt-out, you can use \`ocular telemetry --disable\`\n`
 

packages/ocular/Dockerfile.dev

@@ -4,7 +4,6 @@ FROM node:18-alpine
 RUN apk update && \
   apk add chromium
 
-
 WORKDIR /usr/src/app
 
 RUN npm cache clean --force
@@ -20,7 +19,7 @@ COPY package-lock.json ./
 # Copy the docs package.json
 COPY packages/ocular/package.json ./packages/ocular/package.json
 
-# COPY packages/ocular/env.local ./packages/ocular/.env.local
+COPY env.local ./packages/ocular/.env.local
 
 # Copy turbo.json
 COPY turbo.json ./
@@ -31,12 +30,4 @@ COPY . .
 RUN npm install --verbose
 RUN turbo build
 
-EXPOSE 8080
-
-WORKDIR /usr/src/app/packages/ocular
-
-RUN apk add --no-cache bash curl && curl -1sLf \
-  'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.alpine.sh' | bash \
-  && apk add infisical
-
-CMD ["sh", "-c", "infisical run --env=dev -- npm run start"]
+WORKDIR /usr/src/app/packages/ocular