Remove references to initializeGenkit() in samples (#89)

Author: MichaelDoyle

js/samples/byo-evaluator/src/constants.ts

@@ -0,0 +1,44 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const URL_REGEX =
+  /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/i;
+
+export const US_PHONE_REGEX =
+  /^[\+]?[(]?[0-9]{3}[)]?[-\s\.]?[0-9]{3}[-\s\.]?[0-9]{4}$/i;
+
+// Turn off safety checks for evaluation so that the LLM as an evaluator can
+// respond appropriately to potentially harmful content without error.
+export const PERMISSIVE_SAFETY_SETTINGS: any = {
+  safetySettings: [
+    {
+      category: 'HARM_CATEGORY_HATE_SPEECH',
+      threshold: 'BLOCK_NONE',
+    },
+    {
+      category: 'HARM_CATEGORY_DANGEROUS_CONTENT',
+      threshold: 'BLOCK_NONE',
+    },
+    {
+      category: 'HARM_CATEGORY_HARASSMENT',
+      threshold: 'BLOCK_NONE',
+    },
+    {
+      category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT',
+      threshold: 'BLOCK_NONE',
+    },
+  ],
+};

js/samples/byo-evaluator/src/genkit.config.ts

@@ -15,23 +15,56 @@
  */
 import { EvaluatorAction } from '@genkit-ai/ai';
 import { ModelReference } from '@genkit-ai/ai/model';
-import { PluginProvider, genkitPlugin } from '@genkit-ai/core';
+import { configureGenkit, genkitPlugin, PluginProvider } from '@genkit-ai/core';
+import { firebase } from '@genkit-ai/firebase';
+import { geminiPro, googleAI } from '@genkit-ai/googleai';
 import * as z from 'zod';
 import {
-  DELICIOUSNESS,
+  PERMISSIVE_SAFETY_SETTINGS,
+  URL_REGEX,
+  US_PHONE_REGEX,
+} from './constants.js';
+import {
   createDeliciousnessEvaluator,
+  DELICIOUSNESS,
 } from './deliciousness/deliciousness_evaluator.js';
 import {
-  FUNNINESS,
   createFunninessEvaluator,
+  FUNNINESS,
 } from './funniness/funniness_evaluator.js';
-import { PII_DETECTION, createPiiEvaluator } from './pii/pii_evaluator.js';
+import { createPiiEvaluator, PII_DETECTION } from './pii/pii_evaluator.js';
 import {
-  RegexMetric,
   createRegexEvaluators,
   isRegexMetric,
+  regexMatcher,
+  RegexMetric,
 } from './regex/regex_evaluator.js';
 
+configureGenkit({
+  plugins: [
+    firebase(),
+    googleAI({ apiVersion: ['v1', 'v1beta'] }),
+    byoEval({
+      judge: geminiPro,
+      judgeConfig: PERMISSIVE_SAFETY_SETTINGS,
+      metrics: [
+        // regexMatcher will register an evaluator with a name in the format
+        // byo/regex_match_{suffix}. In this case, byo/regex_match_url
+        regexMatcher('url', URL_REGEX),
+        // byo/regex_match_us_phone
+        regexMatcher('us_phone', US_PHONE_REGEX),
+        PII_DETECTION,
+        DELICIOUSNESS,
+        FUNNINESS,
+      ],
+    }),
+  ],
+  flowStateStore: 'firebase',
+  traceStore: 'firebase',
+  enableTracingAndMetrics: true,
+  logLevel: 'debug',
+});
+
 /**
  * Generic metric definition with flexible configuration.
  */
@@ -92,6 +125,5 @@ export function byoEval<ModelCustomOptions extends z.ZodTypeAny>(
   // create the plugin with the passed params
   return plugin(params);
 }
-export default byoEval;
 
-export * from './genkit.config.js';
+export default byoEval;

js/samples/byo-evaluator/src/index.ts

@@ -14,6 +14,61 @@
  * limitations under the License.
  */
 
-export * from './genkit.config.js';
+import { configureGenkit } from '@genkit-ai/core';
+import { devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore';
+import { genkitEval, GenkitMetric } from '@genkit-ai/evaluator';
+import { firebase } from '@genkit-ai/firebase';
+import { geminiPro, googleAI } from '@genkit-ai/googleai';
+import { textEmbeddingGecko, vertexAI } from '@genkit-ai/vertexai';
+
+// Turn off safety checks for evaluation so that the LLM as an evaluator can
+// respond appropriately to potentially harmful content without error.
+export const PERMISSIVE_SAFETY_SETTINGS: any = {
+  safetySettings: [
+    {
+      category: 'HARM_CATEGORY_HATE_SPEECH',
+      threshold: 'BLOCK_NONE',
+    },
+    {
+      category: 'HARM_CATEGORY_DANGEROUS_CONTENT',
+      threshold: 'BLOCK_NONE',
+    },
+    {
+      category: 'HARM_CATEGORY_HARASSMENT',
+      threshold: 'BLOCK_NONE',
+    },
+    {
+      category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT',
+      threshold: 'BLOCK_NONE',
+    },
+  ],
+};
+
+configureGenkit({
+  plugins: [
+    firebase(),
+    googleAI(),
+    genkitEval({
+      judge: geminiPro,
+      judgeConfig: PERMISSIVE_SAFETY_SETTINGS,
+      metrics: [GenkitMetric.MALICIOUSNESS],
+      embedder: textEmbeddingGecko,
+    }),
+    vertexAI({
+      location: 'us-central1',
+    }),
+    devLocalVectorstore([
+      {
+        indexName: 'pdfQA',
+        embedder: textEmbeddingGecko,
+      },
+    ]),
+  ],
+  flowStateStore: 'firebase',
+  traceStore: 'firebase',
+  enableTracingAndMetrics: true,
+  logLevel: 'debug',
+});
+
 export * from './pdf_rag.js';
 export * from './setup.js';