From 97ef6f24fcf6641b647ebf0dd778d2dd73e7c88a Mon Sep 17 00:00:00 2001
From: Samuel Bushi <ssbushi@google.com>
Date: Thu, 21 Nov 2024 17:37:31 +0000
Subject: [PATCH 1/2] WIP, merge with other sample

---
 .../common/src/eval/localFileDatasetStore.ts     | 12 ++++++++++--
 js/testapps/cat-eval/src/genkit.ts               | 14 +++++++-------
 js/testapps/cat-eval/src/pdf_rag.ts              | 16 ++++++++++++++++
 3 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/genkit-tools/common/src/eval/localFileDatasetStore.ts b/genkit-tools/common/src/eval/localFileDatasetStore.ts
index 79f160873..df9d0f7e6 100644
--- a/genkit-tools/common/src/eval/localFileDatasetStore.ts
+++ b/genkit-tools/common/src/eval/localFileDatasetStore.ts
@@ -160,7 +160,8 @@ export class LocalFileDatasetStore implements DatasetStore {
       throw new Error(`Dataset not found for dataset ID {$id}`);
     }
     return await readFile(filePath, 'utf8').then((data) => {
-      return DatasetSchema.parse(JSON.parse(data));
+      const dataset = DatasetSchema.parse(JSON.parse(data));
+      return dataset;
     });
   }
 
@@ -263,7 +264,14 @@ export class LocalFileDatasetStore implements DatasetStore {
     const patchMap = new Map(patch.map((d) => [d.testCaseId, d]));
 
     patchMap.forEach((value, key) => {
-      datasetMap.set(key, value);
+      // DELETE LOGIC START
+      if (value.testCaseId && !value.input && !value.reference) {
+        datasetMap.delete(key);
+      }
+      // DELETE LOGIC END
+      else {
+        datasetMap.set(key, value);
+      }
     });
 
     const newDataset = Array.from(datasetMap.values()) as Dataset;
diff --git a/js/testapps/cat-eval/src/genkit.ts b/js/testapps/cat-eval/src/genkit.ts
index 580886658..5a40209db 100644
--- a/js/testapps/cat-eval/src/genkit.ts
+++ b/js/testapps/cat-eval/src/genkit.ts
@@ -16,8 +16,11 @@
 
 import { devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore';
 import { genkitEval, GenkitMetric } from '@genkit-ai/evaluator';
-import { gemini15Pro, googleAI } from '@genkit-ai/googleai';
-import { textEmbedding004, vertexAI } from '@genkit-ai/vertexai';
+import {
+  gemini15Pro,
+  googleAI,
+  textEmbeddingGecko001,
+} from '@genkit-ai/googleai';
 import { genkit } from 'genkit';
 
 // Turn off safety checks for evaluation so that the LLM as an evaluator can
@@ -50,15 +53,12 @@ export const ai = genkit({
       judge: gemini15Pro,
       judgeConfig: PERMISSIVE_SAFETY_SETTINGS,
       metrics: [GenkitMetric.MALICIOUSNESS],
-      embedder: textEmbedding004,
-    }),
-    vertexAI({
-      location: 'us-central1',
+      embedder: textEmbeddingGecko001,
     }),
     devLocalVectorstore([
       {
         indexName: 'pdfQA',
-        embedder: textEmbedding004,
+        embedder: textEmbeddingGecko001,
       },
     ]),
   ],
diff --git a/js/testapps/cat-eval/src/pdf_rag.ts b/js/testapps/cat-eval/src/pdf_rag.ts
index d76d2024d..a5ba7bf23 100644
--- a/js/testapps/cat-eval/src/pdf_rag.ts
+++ b/js/testapps/cat-eval/src/pdf_rag.ts
@@ -71,6 +71,22 @@ export const pdfQA = ai.defineFlow(
   }
 );
 
+// Define a simple structured flow, we will evaluate this flow
+export const simpleStructured = ai.defineFlow(
+  {
+    name: 'simpleStructured',
+    inputSchema: z.object({ query: z.string() }),
+    outputSchema: z.object({ response: z.string() }),
+  },
+  async (i) => {
+    const llmResponse = await ai.generate({
+      model: gemini15Flash,
+      prompt: i.query,
+    });
+    return { response: llmResponse.text };
+  }
+);
+
 const chunkingConfig = {
   minLength: 1000, // number of minimum characters into chunk
   maxLength: 2000, // number of maximum characters into chunk

From 9207435e9405307c793549470ea0cdd988faae71 Mon Sep 17 00:00:00 2001
From: Samuel Bushi <ssbushi@google.com>
Date: Mon, 25 Nov 2024 21:30:31 +0000
Subject: [PATCH 2/2] Updaten comments

---
 genkit-tools/common/src/eval/localFileDatasetStore.ts | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/genkit-tools/common/src/eval/localFileDatasetStore.ts b/genkit-tools/common/src/eval/localFileDatasetStore.ts
index df9d0f7e6..95c08074d 100644
--- a/genkit-tools/common/src/eval/localFileDatasetStore.ts
+++ b/genkit-tools/common/src/eval/localFileDatasetStore.ts
@@ -160,8 +160,7 @@ export class LocalFileDatasetStore implements DatasetStore {
       throw new Error(`Dataset not found for dataset ID {$id}`);
     }
     return await readFile(filePath, 'utf8').then((data) => {
-      const dataset = DatasetSchema.parse(JSON.parse(data));
-      return dataset;
+      return DatasetSchema.parse(JSON.parse(data));
     });
   }
 
@@ -264,12 +263,10 @@ export class LocalFileDatasetStore implements DatasetStore {
     const patchMap = new Map(patch.map((d) => [d.testCaseId, d]));
 
     patchMap.forEach((value, key) => {
-      // DELETE LOGIC START
+      // Delete sample if testCaseId is provided
       if (value.testCaseId && !value.input && !value.reference) {
         datasetMap.delete(key);
-      }
-      // DELETE LOGIC END
-      else {
+      } else {
         datasetMap.set(key, value);
       }
     });