feat(js/plugins/ollama): Ollama embeddings (#807)

* feat: do not allow defining new actions from within other actions/flows (#725)

* refactor: [JS] introduced a generate utility action to make generate veneer logic reusable (#759)

Co-authored-by: Michael Bleigh <bleigh@google.com>

* refactor: consolidated registry into a class, made registry hierarchical (#639)

* Add DatasetStore interfaces for evals datasets (#781)

* [Fix] Apply #777 to `next` branch (#779)

* Merge main to Next (#792)

* Update index.md (#764)

* test(go): add live tests to go ollama plugin (#720)

* test(go): add ollama live test

* test(go): retrieve port info for test container

* chore(go): refactor ollama plugin live test

* test(go): remove docker from ollama live test

* fix context array (#777)

* Update to pnpm v9.7.1 (#786)

---------

Co-authored-by: Peter Friese <peter@peterfriese.de>
Co-authored-by: Jacob Cable <32874567+cabljac@users.noreply.github.com>
Co-authored-by: ssbushi <66321939+ssbushi@users.noreply.github.com>
Co-authored-by: Anthony Barone <tonybaroneee@gmail.com>

* [Eval] Breaking change -- Modify EvalRunKey (#755)

* Makes inputSchema optional for tools. (#822)

* Add datasetId field to eval run key (#827)

* Fixes history rendering of Dotprompt system role. (#837)

* Local file based DatasetStore implementation and Tools API changes (#797)

* feat(js/plugins/ollama): add initial embedding support

* feat(js/plugins/ollama): integrate ollama embeddings into plugin proper

* Update js/plugins/ollama/src/embeddings.ts

Co-authored-by: Pavel Jbanov <pavelj@google.com>

---------

Co-authored-by: Pavel Jbanov <pavelj@google.com>
Co-authored-by: Michael Bleigh <bleigh@google.com>
Co-authored-by: shrutip90 <shruti.p90@gmail.com>
Co-authored-by: ssbushi <66321939+ssbushi@users.noreply.github.com>
Co-authored-by: Sam Phillips <samphillips@google.com>
Co-authored-by: Peter Friese <peter@peterfriese.de>
Co-authored-by: Anthony Barone <tonybaroneee@gmail.com>
Co-authored-by: Michael Doyle <michaeldoyle@google.com>

docs/errors/no_new_actions_at_runtime.md

@@ -0,0 +1,22 @@
+# No new actions at runtime error
+
+Defining new actions at runtime is not allowed.
+
+✅ DO:
+
+```ts
+const prompt = defineDotprompt({...})
+
+const flow = defineFlow({...}, async (input) => {
+ await prompt.generate(...);
+})
+```
+
+❌ DON'T:
+
+```ts
+const flow = defineFlow({...}, async (input) => {
+ const prompt = defineDotprompt({...})
+ prompt.generate(...);
+})
+```

genkit-tools/cli/src/commands/eval-flow.ts

@@ -180,7 +180,7 @@ export const evalFlow = new Command('eval:flow')
 
  const evalRun = {
  key: {
- actionId: flowName,
+ actionRef: `/flow/${flowName}`,
  evalRunId,
  createdAt: new Date().toISOString(),
  },

genkit-tools/common/src/eval/index.ts

@@ -14,7 +14,8 @@
  * limitations under the License.
  */
 
-import { EvalStore } from '../types/eval';
+import { DatasetStore, EvalStore } from '../types/eval';
+import { LocalFileDatasetStore } from './localFileDatasetStore';
 import { LocalFileEvalStore } from './localFileEvalStore';
 export { EvalFlowInput, EvalFlowInputSchema } from '../types/eval';
 export * from './exporter';
@@ -24,3 +25,7 @@ export function getEvalStore(): EvalStore {
  // TODO: This should provide EvalStore, based on tools config.
  return LocalFileEvalStore.getEvalStore();
 }
+
+export function getDatasetStore(): DatasetStore {
+ return LocalFileDatasetStore.getDatasetStore();
+}

genkit-tools/common/src/eval/localFileDatasetStore.ts

@@ -0,0 +1,222 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import crypto from 'crypto';
+import fs from 'fs';
+import { readFile, rm, writeFile } from 'fs/promises';
+import path from 'path';
+import { v4 as uuidv4 } from 'uuid';
+import { CreateDatasetRequest, UpdateDatasetRequest } from '../types/apis';
+import {
+ Dataset,
+ DatasetMetadata,
+ DatasetStore,
+ EvalFlowInputSchema,
+} from '../types/eval';
+import { logger } from '../utils/logger';
+
+/**
+ * A local, file-based DatasetStore implementation.
+ */
+export class LocalFileDatasetStore implements DatasetStore {
+ private readonly storeRoot;
+ private readonly indexFile;
+ private readonly INDEX_DELIMITER = '\n';
+ private static cachedDatasetStore: LocalFileDatasetStore | null = null;
+
+ private constructor(storeRoot: string) {
+ this.storeRoot = storeRoot;
+ this.indexFile = this.getIndexFilePath();
+ fs.mkdirSync(this.storeRoot, { recursive: true });
+ if (!fs.existsSync(this.indexFile)) {
+ fs.writeFileSync(path.resolve(this.indexFile), '');
+ }
+ logger.info(
+ `Initialized local file dataset store at root: ${this.storeRoot}`
+ );
+ }
+
+ static getDatasetStore() {
+ if (!this.cachedDatasetStore) {
+ this.cachedDatasetStore = new LocalFileDatasetStore(
+ this.generateRootPath()
+ );
+ }
+ return this.cachedDatasetStore;
+ }
+
+ static reset() {
+ this.cachedDatasetStore = null;
+ }
+
+ async createDataset(req: CreateDatasetRequest): Promise<DatasetMetadata> {
+ return this.createDatasetInternal(req.data, req.displayName);
+ }
+
+ private async createDatasetInternal(
+ data: Dataset,
+ displayName?: string
+ ): Promise<DatasetMetadata> {
+ const datasetId = this.generateDatasetId();
+ const filePath = path.resolve(
+ this.storeRoot,
+ this.generateFileName(datasetId)
+ );
+
+ if (fs.existsSync(filePath)) {
+ logger.error(`Dataset already exists at ` + filePath);
+ throw new Error(
+ `Create dataset failed: file already exists at {$filePath}`
+ );
+ }
+
+ logger.info(`Saving Dataset to ` + filePath);
+ await writeFile(filePath, JSON.stringify(data));
+
+ const now = new Date().toString();
+ const metadata = {
+ datasetId,
+ size: Array.isArray(data) ? data.length : data.samples.length,
+ version: 1,
+ displayName: displayName,
+ createTime: now,
+ updateTime: now,
+ };
+
+ let metadataMap = await this.getMetadataMap();
+ metadataMap[datasetId] = metadata;
+
+ logger.debug(
+ `Saving DatasetMetadata for ID ${datasetId} to ` +
+ path.resolve(this.indexFile)
+ );
+
+ await writeFile(path.resolve(this.indexFile), JSON.stringify(metadataMap));
+ return metadata;
+ }
+
+ async updateDataset(req: UpdateDatasetRequest): Promise<DatasetMetadata> {
+ const datasetId = req.datasetId;
+ const filePath = path.resolve(
+ this.storeRoot,
+ this.generateFileName(datasetId)
+ );
+ if (!fs.existsSync(filePath)) {
+ throw new Error(`Update dataset failed: dataset not found`);
+ }
+
+ let metadataMap = await this.getMetadataMap();
+ const prevMetadata = metadataMap[datasetId];
+ if (!prevMetadata) {
+ throw new Error(`Update dataset failed: dataset metadata not found`);
+ }
+
+ logger.info(`Updating Dataset at ` + filePath);
+ await writeFile(filePath, JSON.stringify(req.patch));
+
+ const now = new Date().toString();
+ const newMetadata = {
+ datasetId: datasetId,
+ size: Array.isArray(req.patch)
+ ? req.patch.length
+ : req.patch.samples.length,
+ version: prevMetadata.version + 1,
+ displayName: req.displayName,
+ createTime: prevMetadata.createTime,
+ updateTime: now,
+ };
+
+ logger.debug(
+ `Updating DatasetMetadata for ID ${datasetId} at ` +
+ path.resolve(this.indexFile)
+ );
+ // Replace the metadata object in the metadata map
+ metadataMap[datasetId] = newMetadata;
+ await writeFile(path.resolve(this.indexFile), JSON.stringify(metadataMap));
+
+ return newMetadata;
+ }
+
+ async getDataset(datasetId: string): Promise<Dataset> {
+ const filePath = path.resolve(
+ this.storeRoot,
+ this.generateFileName(datasetId)
+ );
+ if (!fs.existsSync(filePath)) {
+ throw new Error(`Dataset not found for dataset ID {$id}`);
+ }
+ return await readFile(filePath, 'utf8').then((data) =>
+ EvalFlowInputSchema.parse(JSON.parse(data))
+ );
+ }
+
+ async listDatasets(): Promise<DatasetMetadata[]> {
+ return this.getMetadataMap().then((metadataMap) => {
+ let metadatas = [];
+
+ for (var key in metadataMap) {
+ metadatas.push(metadataMap[key]);
+ }
+ return metadatas;
+ });
+ }
+
+ async deleteDataset(datasetId: string): Promise<void> {
+ const filePath = path.resolve(
+ this.storeRoot,
+ this.generateFileName(datasetId)
+ );
+ await rm(filePath);
+
+ let metadataMap = await this.getMetadataMap();
+ delete metadataMap[datasetId];
+
+ logger.debug(
+ `Deleting DatasetMetadata for ID ${datasetId} in ` +
+ path.resolve(this.indexFile)
+ );
+ await writeFile(path.resolve(this.indexFile), JSON.stringify(metadataMap));
+ }
+
+ private static generateRootPath(): string {
+ const rootHash = crypto
+ .createHash('md5')
+ .update(process.cwd() || 'unknown')
+ .digest('hex');
+ return path.resolve(process.cwd(), `.genkit/${rootHash}/datasets`);
+ }
+
+ private generateDatasetId(): string {
+ return uuidv4();
+ }
+
+ private generateFileName(datasetId: string): string {
+ return `${datasetId}.json`;
+ }
+
+ private getIndexFilePath(): string {
+ return path.resolve(this.storeRoot, 'index.json');
+ }
+
+ private async getMetadataMap(): Promise<any> {
+ if (!fs.existsSync(this.indexFile)) {
+ return Promise.resolve({} as any);
+ }
+ return await readFile(path.resolve(this.indexFile), 'utf8').then((data) =>
+ JSON.parse(data)
+ );
+ }
+}

genkit-tools/common/src/eval/localFileEvalStore.ts

@@ -61,10 +61,7 @@ export class LocalFileEvalStore implements EvalStore {
  }
 
  async save(evalRun: EvalRun): Promise<void> {
- const fileName = this.generateFileName(
- evalRun.key.evalRunId,
- evalRun.key.actionId
- );
+ const fileName = this.generateFileName(evalRun.key.evalRunId);
 
  logger.info(
  `Saving EvalRun ${evalRun.key.evalRunId} to ` +
@@ -85,13 +82,10 @@ export class LocalFileEvalStore implements EvalStore {
  );
  }
 
- async load(
- evalRunId: string,
- actionId?: string
- ): Promise<EvalRun | undefined> {
+ async load(evalRunId: string): Promise<EvalRun | undefined> {
  const filePath = path.resolve(
  this.storeRoot,
- this.generateFileName(evalRunId, actionId)
+ this.generateFileName(evalRunId)
  );
  if (!fs.existsSync(filePath)) {
  return undefined;
@@ -117,8 +111,8 @@ export class LocalFileEvalStore implements EvalStore {
 
  logger.debug(`Found keys: ${JSON.stringify(keys)}`);
 
- if (query?.filter?.actionId) {
- keys = keys.filter((key) => key.actionId === query?.filter?.actionId);
+ if (query?.filter?.actionRef) {
+ keys = keys.filter((key) => key.actionRef === query?.filter?.actionRef);
  logger.debug(`Filtered keys: ${JSON.stringify(keys)}`);
  }
 
@@ -127,12 +121,8 @@ export class LocalFileEvalStore implements EvalStore {
  };
  }
 
- private generateFileName(evalRunId: string, actionId?: string): string {
- if (!actionId) {
- return `${evalRunId}.json`;
- }
-
- return `${actionId?.replace('/', '_')}-${evalRunId}.json`;
+ private generateFileName(evalRunId: string): string {
+ return `${evalRunId}.json`;
  }
 
  private getIndexFilePath(): string {