diff --git a/docs-go/Makefile b/docs-go/Makefile index 664d86515..208463e25 100644 --- a/docs-go/Makefile +++ b/docs-go/Makefile @@ -2,7 +2,7 @@ WEAVE=$(HOME)/go/bin/weave all: $(WEAVE) get-started-go.md flows.md models.md prompts.md dotprompt.md pgvector.md \ cloud-run.md rag.md \ - plugins/ollama.md + plugins/ollama.md plugins/pinecone.md $(WEAVE): ../go/internal/cmd/weave/*.go go -C ../go install ./internal/cmd/weave diff --git a/docs-go/plugins/pinecone.md b/docs-go/plugins/pinecone.md new file mode 100644 index 000000000..a9313fb33 --- /dev/null +++ b/docs-go/plugins/pinecone.md @@ -0,0 +1,95 @@ + + +# Pinecone plugin + +The Pinecone plugin provides indexer and retriever implementatons that use the +[Pinecone](https://www.pinecone.io/) cloud vector database. + +## Configuration + +To use this plugin, import the `pinecone` package and call `pinecone.Init()`: + +```go +import "github.com/firebase/genkit/go/plugins/pinecone" +``` + +```go +if err := pinecone.Init(ctx, ""); err != nil { + return err +} +``` + +The plugin requires your Pinecone API key. +Configure the plugin to use your API key by doing one of the following: + +- Set the `PINECONE_API_KEY` environment variable to your API key. + +- Specify the API key when you initialize the plugin: + + ```go + if err := pinecone.Init(ctx, pineconeAPIKey); err != nil { + return err + } + ``` + + However, don't embed your API key directly in code! Use this feature only + in conjunction with a service like Cloud Secret Manager or similar. + +## Usage + +To add documents to a Pinecone index, first create an index definition that +specifies the name of the index and the embedding model you're using: + +```go +menuIndexer, err := pinecone.DefineIndexer(ctx, pinecone.Config{ + IndexID: "menu_data", // Your Pinecone index + Embedder: googleai.Embedder("text-embedding-004"), // Embedding model of your choice +}) +if err != nil { + return err +} +``` + +You can also optionally specify the key that Pinecone uses for document data +(`_content`, by default). + +Then, call the index's `Index()` method, passing it a list of the documents you +want to add: + +```go +if err := menuIndexer.Index( + ctx, + &ai.IndexerRequest{Documents: docChunks, Options: nil}, +); err != nil { + return err +} +``` + +Similarly, to retrieve documents from an index, first create a retriever +definition: + +```go +menuRetriever, err := pinecone.DefineRetriever(ctx, pinecone.Config{ + IndexID: "menu_data", // Your Pinecone index + Embedder: googleai.Embedder("text-embedding-004"), // Embedding model of your choice +}) +if err != nil { + return err +} +``` + +Then, call the retriever's `Retrieve()` method, passing it a text query: + +```go +resp, err := menuRetriever.Retrieve(ctx, &ai.RetrieverRequest{ + Document: ai.DocumentFromText(userInput, nil), + Options: nil, +}) +if err != nil { + return err +} +menuInfo := resp.Documents +``` + +See the [Retrieval-augmented generation](../rag.md) page for a general +discussion on using indexers and retrievers for RAG. diff --git a/docs-go/plugins/pinecone.src b/docs-go/plugins/pinecone.src new file mode 100644 index 000000000..f2f301924 --- /dev/null +++ b/docs-go/plugins/pinecone.src @@ -0,0 +1,53 @@ +# Pinecone plugin + +The Pinecone plugin provides indexer and retriever implementatons that use the +[Pinecone](https://www.pinecone.io/) cloud vector database. + +## Configuration + +To use this plugin, import the `pinecone` package and call `pinecone.Init()`: + +```go +import "github.com/firebase/genkit/go/plugins/pinecone" +``` + +%include ../go/internal/doc-snippets/pinecone.go init + +The plugin requires your Pinecone API key. +Configure the plugin to use your API key by doing one of the following: + +- Set the `PINECONE_API_KEY` environment variable to your API key. + +- Specify the API key when you initialize the plugin: + + %include ../go/internal/doc-snippets/pinecone.go initkey + + However, don't embed your API key directly in code! Use this feature only + in conjunction with a service like Cloud Secret Manager or similar. + +## Usage + +To add documents to a Pinecone index, first create an index definition that +specifies the name of the index and the embedding model you're using: + +%include ../go/internal/doc-snippets/pinecone.go defineindex + +You can also optionally specify the key that Pinecone uses for document data +(`_content`, by default). + +Then, call the index's `Index()` method, passing it a list of the documents you +want to add: + +%include ../go/internal/doc-snippets/pinecone.go index + +Similarly, to retrieve documents from an index, first create a retriever +definition: + +%include ../go/internal/doc-snippets/pinecone.go defineretriever + +Then, call the retriever's `Retrieve()` method, passing it a text query: + +%include ../go/internal/doc-snippets/pinecone.go retrieve + +See the [Retrieval-augmented generation](../rag.md) page for a general +discussion on using indexers and retrievers for RAG. diff --git a/go/internal/doc-snippets/pinecone.go b/go/internal/doc-snippets/pinecone.go new file mode 100644 index 000000000..9a53e06d0 --- /dev/null +++ b/go/internal/doc-snippets/pinecone.go @@ -0,0 +1,88 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snippets + +import ( + "context" + + "github.com/firebase/genkit/go/ai" + "github.com/firebase/genkit/go/plugins/googleai" + "github.com/firebase/genkit/go/plugins/pinecone" +) + +func pineconeEx(ctx context.Context) error { + var err error + + //!+init + if err := pinecone.Init(ctx, ""); err != nil { + return err + } + //!-init + + var pineconeAPIKey string + //!+initkey + if err := pinecone.Init(ctx, pineconeAPIKey); err != nil { + return err + } + //!-initkey + + //!+defineindex + menuIndexer, err := pinecone.DefineIndexer(ctx, pinecone.Config{ + IndexID: "menu_data", // Your Pinecone index + Embedder: googleai.Embedder("text-embedding-004"), // Embedding model of your choice + }) + if err != nil { + return err + } + //!-defineindex + + var docChunks []*ai.Document + + //!+index + if err := menuIndexer.Index( + ctx, + &ai.IndexerRequest{Documents: docChunks, Options: nil}, + ); err != nil { + return err + } + //!-index + + //!+defineretriever + menuRetriever, err := pinecone.DefineRetriever(ctx, pinecone.Config{ + IndexID: "menu_data", // Your Pinecone index + Embedder: googleai.Embedder("text-embedding-004"), // Embedding model of your choice + }) + if err != nil { + return err + } + //!-defineretriever + + var userInput string + + //!+retrieve + resp, err := menuRetriever.Retrieve(ctx, &ai.RetrieverRequest{ + Document: ai.DocumentFromText(userInput, nil), + Options: nil, + }) + if err != nil { + return err + } + menuInfo := resp.Documents + //!-retrieve + + _ = menuInfo + + return nil +}