insert human eval scores to clickhouse (#163)

* insert human eval scores to clickhouse

* don't setup env vars

docker-compose-full.yml

@@ -119,6 +119,8 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
+      clickhouse:
+        condition: service_started
     environment:
       - PORT=3000
       - BACKEND_URL=http://app-server:8000
@@ -128,6 +130,8 @@ services:
       - NEXTAUTH_SECRET=some_secret
       - NEXT_PUBLIC_URL=http://localhost:3000
       - ENVIRONMENT=FULL
+      - CLICKHOUSE_URL=http://clickhouse:8123
+      - CLICKHOUSE_USER=${CLICKHOUSE_USER}
 
 volumes:
   qdrant-data:

docker-compose-local-build.yml

@@ -126,12 +126,16 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
+      clickhouse:
+        condition: service_started
     environment:
       - PORT=3000
       - BACKEND_URL=http://app-server:8000
       - SHARED_SECRET_TOKEN=${SHARED_SECRET_TOKEN}
       - DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
       - ENVIRONMENT=FULL
+      - CLICKHOUSE_URL=http://clickhouse:8123
+      - CLICKHOUSE_USER=${CLICKHOUSE_USER}
 
 volumes:
   qdrant-data:

frontend/app/api/projects/[projectId]/queues/[queueId]/remove/route.ts

@@ -1,7 +1,10 @@
 import { isCurrentUserMemberOfProject } from '@/lib/db/utils';
 import { db } from '@/lib/db/drizzle';
-import { labelingQueueItems, labels, evaluationScores, labelingQueues } from '@/lib/db/migrations/schema';
+import { labelingQueueItems, labels, evaluationScores, labelingQueues, evaluations, evaluationResults } from '@/lib/db/migrations/schema';
 import { eq, and } from 'drizzle-orm';
+import { isFeatureEnabled } from '@/lib/features/features';
+import { Feature } from '@/lib/features/features';
+import { clickhouseClient } from '@/lib/clickhouse/client';
 
 
 // remove an item from the queue
@@ -14,12 +17,13 @@ export async function POST(request: Request, { params }: { params: { projectId:
   const { id, spanId, action } = body;
 
   if (action.resultId) {
-
     const labelingQueue = await db.query.labelingQueues.findFirst({
       where: eq(labelingQueues.id, params.queueId)
     });
 
     // get all labels of the span
+    // FIXME: this takes values from previous labels,
+    // potentially from a different queue
     const labelsOfSpan = await db.query.labels.findMany({
       where: eq(labels.spanId, spanId),
       with: {
@@ -36,6 +40,30 @@ export async function POST(request: Request, { params }: { params: { projectId:
       resultId,
     }));
 
+    if (isFeatureEnabled(Feature.FULL_BUILD)) {
+      const evaluation = await db.query.evaluations.findFirst({
+        with: {
+          evaluationResults: {
+            where: eq(evaluationResults.id, resultId)
+          }
+        }
+      });
+      if (evaluation && evaluationValues.length > 0) {
+        const result = await clickhouseClient.insert({
+          table: 'evaluation_scores',
+          format: 'JSONEachRow',
+          values: evaluationValues.map(value => ({
+            project_id: params.projectId,
+            group_id: evaluation.groupId,
+            evaluation_id: evaluation.id,
+            result_id: resultId,
+            name: value.name,
+            value: value.score,
+          }))
+        });
+      }
+    }
+
     await db.insert(evaluationScores).values(evaluationValues);
   }
 

frontend/lib/clickhouse/client.ts

@@ -0,0 +1,17 @@
+import { createClient } from "@clickhouse/client";
+import { config } from "dotenv";
+
+config({ path: ".env" }); // or .env.local
+
+// https://clickhouse.com/docs/en/cloud/bestpractices/asynchronous-inserts -> Create client which will wait for async inserts
+// For now, we're not waiting for inserts to finish, but later need to add queue and batch on client-side
+export const clickhouseClient = createClient({
+  url: process.env.CLICKHOUSE_URL,
+  username: process.env.CLICKHOUSE_USER,
+  password: process.env.CLICKHOUSE_PASSWORD ?? "",
+  database: "default",
+  clickhouse_settings: {
+    async_insert: 1,
+    wait_for_async_insert: 0,
+  },
+});

frontend/package.json

@@ -15,6 +15,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.674.0",
+    "@clickhouse/client": "^1.7.0",
     "@codemirror/lang-json": "^6.0.1",
     "@codemirror/lang-python": "^6.1.6",
     "@codemirror/lang-yaml": "^6.1.1",