fix(cli): remove datataset sample if only testCaseId is provided (#1398)

firebase · Nov 27, 2024 · 5395b9b · 5395b9b
1 parent 01f1bde
commit 5395b9b
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 6 deletions.
diff --git a/genkit-tools/common/src/eval/localFileDatasetStore.ts b/genkit-tools/common/src/eval/localFileDatasetStore.ts
@@ -263,7 +263,12 @@ export class LocalFileDatasetStore implements DatasetStore {
     const patchMap = new Map(patch.map((d) => [d.testCaseId, d]));
 
     patchMap.forEach((value, key) => {
-      datasetMap.set(key, value);
+      // Delete sample if testCaseId is provided
+      if (value.testCaseId && !value.input && !value.reference) {
+        datasetMap.delete(key);
+      } else {
+        datasetMap.set(key, value);
+      }
     });
 
     const newDataset = Array.from(datasetMap.values()) as Dataset;

diff --git a/js/testapps/evals/src/genkit.ts b/js/testapps/evals/src/genkit.ts
@@ -16,8 +16,13 @@
 
 import { devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore';
 import { genkitEval, GenkitMetric } from '@genkit-ai/evaluator';
-import { gemini15Flash, gemini15Pro, googleAI } from '@genkit-ai/googleai';
-import { textEmbedding004, vertexAI } from '@genkit-ai/vertexai';
+import {
+  gemini15Flash,
+  gemini15Pro,
+  googleAI,
+  textEmbeddingGecko001,
+} from '@genkit-ai/googleai';
+import { vertexAI } from '@genkit-ai/vertexai';
 import {
   vertexAIEvaluation,
   VertexAIEvaluationMetricType,
@@ -54,8 +59,8 @@ export const ai = genkit({
     genkitEval({
       judge: gemini15Pro,
       judgeConfig: PERMISSIVE_SAFETY_SETTINGS,
-      metrics: [GenkitMetric.FAITHFULNESS, GenkitMetric.MALICIOUSNESS],
-      embedder: textEmbedding004,
+      metrics: [GenkitMetric.MALICIOUSNESS],
+      embedder: textEmbeddingGecko001,
     }),
     vertexAI({
       location: 'us-central1',
@@ -79,7 +84,7 @@ export const ai = genkit({
     devLocalVectorstore([
       {
         indexName: 'pdfQA',
-        embedder: textEmbedding004,
+        embedder: textEmbeddingGecko001,
       },
     ]),
     langchain({

diff --git a/js/testapps/evals/src/pdf-rag.ts b/js/testapps/evals/src/pdf-rag.ts
@@ -71,6 +71,22 @@ export const pdfQA = ai.defineFlow(
   }
 );
 
+// Define a simple structured flow, we will evaluate this flow
+export const simpleStructured = ai.defineFlow(
+  {
+    name: 'simpleStructured',
+    inputSchema: z.object({ query: z.string() }),
+    outputSchema: z.object({ response: z.string() }),
+  },
+  async (i) => {
+    const llmResponse = await ai.generate({
+      model: gemini15Flash,
+      prompt: i.query,
+    });
+    return { response: llmResponse.text };
+  }
+);
+
 const chunkingConfig = {
   minLength: 1000, // number of minimum characters into chunk
   maxLength: 2000, // number of maximum characters into chunk