From 2a1f31ff467f454a5ded6a746117a34f3043e2e6 Mon Sep 17 00:00:00 2001
From: Tunde Agboola <tun.agboola@gmail.com>
Date: Wed, 1 May 2024 17:00:35 -0400
Subject: [PATCH] Add vertex evaluator information to docs

---
 docs/plugins/vertex-ai.md             | 44 ++++++++++++++++++++++++++-
 js/plugins/vertexai/src/evaluation.ts |  1 +
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/docs/plugins/vertex-ai.md b/docs/plugins/vertex-ai.md
index b5b7de818..dd984b5d2 100644
--- a/docs/plugins/vertex-ai.md
+++ b/docs/plugins/vertex-ai.md
@@ -7,6 +7,13 @@ through the [Vertex AI API](https://cloud.google.com/vertex-ai/generative-ai/doc
 - Imagen2 image generation
 - Gecko text embedding generation
 
+It also provides access to subset of evaluation metrics through the Vertex AI [Rapid Evaluation API](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation).
+
+- [Safety](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#safetyinput)
+- [Groundeness](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#groundednessinput)
+- [ROUGE](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#rougeinput)
+- [BLEU](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#bleuinput)
+
 ## Installation
 
 ```posix-terminal
@@ -66,6 +73,8 @@ credentials.
 
 ## Usage
 
+### Generative AI Models
+
 This plugin statically exports references to its supported generative AI models:
 
 ```js
@@ -115,7 +124,7 @@ const embedding = await embed({
 });
 ```
 
-### Anthropic Claude 3 on Vertex AI Model Garden
+#### Anthropic Claude 3 on Vertex AI Model Garden
 
 If you have access to Claude 3 models ([haiku](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-haiku), [sonnet](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-sonnet) or [opus](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-opus)) in Vertex AI Model Garden you can use them with Genkit.
 
@@ -147,3 +156,36 @@ const llmResponse = await generate({
   prompt: 'What should I do when I visit Melbourne?',
 });
 ```
+
+### Evaluators
+
+To use the evaluators from Vertex AI Rapid Evaluation, add an `evaluation` block to your `vertexAI` plugin configuration.
+
+```js
+import { vertexAI, VertexAIEvaluationMetricType } from '@genkit-ai/vertexai';
+
+export default configureGenkit({
+  plugins: [
+    vertexAI({
+      projectId: 'your-cloud-project',
+      location: 'us-central1',
+      evaluation: {
+        metrics: [
+          VertexAIEvaluationMetricType.SAFETY,
+          {
+            type: VertexAIEvaluationMetricType.ROUGE,
+            metricSpec: {
+              rougeType: 'rougeLsum',
+            },
+          },
+        ],
+      },
+    }),
+  ],
+  // ...
+});
+```
+
+The configuration above adds evaluators for the `Safety` and `ROUGE` metrics. The example shows two approaches- the `Safety` metric uses the default specification, whereas the `ROUGE` metric provides a customized specification that sets the rouge type to `rougeLsum`.
+
+Both evaluators can be run using the `genkit eval:run` with a compatible dataset i.e. a datset with `output` and `reference` fields. The `Safety` evaluator can also be run using the `genkit eval:flow -e vertexai/safety` command since it only requires an `output`.
diff --git a/js/plugins/vertexai/src/evaluation.ts b/js/plugins/vertexai/src/evaluation.ts
index c8fa1937e..f5a9195e8 100644
--- a/js/plugins/vertexai/src/evaluation.ts
+++ b/js/plugins/vertexai/src/evaluation.ts
@@ -25,6 +25,7 @@ import { EvaluatorFactory } from './evaluator_factory';
  * https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation#parameter-list
  */
 export enum VertexAIEvaluationMetricType {
+  // Update genkit/docs/plugins/vertex-ai.md when modifying the list of enums
   SAFETY = 'SAFETY',
   GROUNDEDNESS = 'GROUNDEDNESS',
   BLEU = 'BLEU',