From 266276bf62f6e2f8dc5f2a01ab9df39db3f13b2c Mon Sep 17 00:00:00 2001 From: Alex Pascal Date: Tue, 9 Sep 2025 20:36:56 -0700 Subject: [PATCH 1/4] Update embedder.go --- go/ai/embedder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/ai/embedder.go b/go/ai/embedder.go index 9365c2bbea..896ff2e617 100644 --- a/go/ai/embedder.go +++ b/go/ai/embedder.go @@ -127,7 +127,7 @@ func NewEmbedder(name string, opts *EmbedderOptions, fn EmbedderFunc) Embedder { } return &embedder{ - ActionDef: *core.NewAction(name, api.ActionTypeEmbedder, metadata, nil, fn), + ActionDef: *core.NewAction(name, api.ActionTypeEmbedder, metadata, inputSchema, fn), } } From 4fc5f5e3dff170c2e80617ec68fde22841e1d5dc Mon Sep 17 00:00:00 2001 From: Alex Pascal Date: Wed, 10 Sep 2025 12:17:00 -0700 Subject: [PATCH 2/4] Fixed custom options schema. --- go/ai/embedder.go | 4 +- go/ai/evaluator.go | 4 +- go/ai/generate.go | 4 +- go/ai/retriever.go | 4 +- go/internal/base/json.go | 2 +- go/plugins/googlegenai/gemini.go | 3 - go/plugins/googlegenai/googlegenai.go | 279 ++++++++++++-------------- go/plugins/googlegenai/models.go | 51 ----- 8 files changed, 138 insertions(+), 213 deletions(-) diff --git a/go/ai/embedder.go b/go/ai/embedder.go index 896ff2e617..b84e3df81b 100644 --- a/go/ai/embedder.go +++ b/go/ai/embedder.go @@ -121,8 +121,8 @@ func NewEmbedder(name string, opts *EmbedderOptions, fn EmbedderFunc) Embedder { inputSchema := core.InferSchemaMap(EmbedRequest{}) if inputSchema != nil && opts.ConfigSchema != nil { - if _, ok := inputSchema["options"]; ok { - inputSchema["options"] = opts.ConfigSchema + if props, ok := inputSchema["properties"].(map[string]any); ok { + props["options"] = opts.ConfigSchema } } diff --git a/go/ai/evaluator.go b/go/ai/evaluator.go index d628831933..0fb470e6db 100644 --- a/go/ai/evaluator.go +++ b/go/ai/evaluator.go @@ -184,8 +184,8 @@ func NewEvaluator(name string, opts *EvaluatorOptions, fn EvaluatorFunc) Evaluat inputSchema := core.InferSchemaMap(EvaluatorRequest{}) if inputSchema != nil && opts.ConfigSchema != nil { - if _, ok := inputSchema["options"]; ok { - inputSchema["options"] = opts.ConfigSchema + if props, ok := inputSchema["properties"].(map[string]any); ok { + props["options"] = opts.ConfigSchema } } diff --git a/go/ai/generate.go b/go/ai/generate.go index d54a09e4b5..83a1784066 100644 --- a/go/ai/generate.go +++ b/go/ai/generate.go @@ -173,8 +173,8 @@ func NewModel(name string, opts *ModelOptions, fn ModelFunc) Model { inputSchema := core.InferSchemaMap(ModelRequest{}) if inputSchema != nil && opts.ConfigSchema != nil { - if _, ok := inputSchema["config"]; ok { - inputSchema["config"] = opts.ConfigSchema + if props, ok := inputSchema["properties"].(map[string]any); ok { + props["config"] = opts.ConfigSchema } } diff --git a/go/ai/retriever.go b/go/ai/retriever.go index 804037c71a..f664de4954 100644 --- a/go/ai/retriever.go +++ b/go/ai/retriever.go @@ -115,8 +115,8 @@ func NewRetriever(name string, opts *RetrieverOptions, fn RetrieverFunc) Retriev inputSchema := core.InferSchemaMap(RetrieverRequest{}) if inputSchema != nil && opts.ConfigSchema != nil { - if _, ok := inputSchema["options"]; ok { - inputSchema["options"] = opts.ConfigSchema + if props, ok := inputSchema["properties"].(map[string]any); ok { + props["options"] = opts.ConfigSchema } } diff --git a/go/internal/base/json.go b/go/internal/base/json.go index cceff1f0b0..a891c765e0 100644 --- a/go/internal/base/json.go +++ b/go/internal/base/json.go @@ -94,8 +94,8 @@ func InferJSONSchema(x any) (s *jsonschema.Schema) { }, } s = r.Reflect(x) - // TODO: Unwind this change once Monaco Editor supports newer than JSON schema draft-07. s.Version = "" + s.ID = "" return s } diff --git a/go/plugins/googlegenai/gemini.go b/go/plugins/googlegenai/gemini.go index 88961d26d7..a131704214 100644 --- a/go/plugins/googlegenai/gemini.go +++ b/go/plugins/googlegenai/gemini.go @@ -40,9 +40,6 @@ import ( ) const ( - // Thinking budget limit - thinkingBudgetMax = 24576 - // Tool name regex toolNameRegex = "^[a-zA-Z_][a-zA-Z0-9_.-]{0,63}$" ) diff --git a/go/plugins/googlegenai/googlegenai.go b/go/plugins/googlegenai/googlegenai.go index 756ae8fda6..396a03d4a8 100644 --- a/go/plugins/googlegenai/googlegenai.go +++ b/go/plugins/googlegenai/googlegenai.go @@ -28,6 +28,27 @@ const ( vertexAILabelPrefix = "Vertex AI" ) +var ( + defaultGeminiOpts = ai.ModelOptions{ + Supports: &Multimodal, + Versions: []string{}, + Stage: ai.ModelStageUnstable, + } + + defaultImagenOpts = ai.ModelOptions{ + Supports: &Media, + Versions: []string{}, + Stage: ai.ModelStageUnstable, + } + + defaultEmbedOpts = ai.EmbedderOptions{ + Supports: &ai.EmbedderSupports{ + Input: []string{"text"}, + }, + Dimensions: 768, + } +) + // GoogleAI is a Genkit plugin for interacting with the Google AI service. type GoogleAI struct { APIKey string // API key to access the service. If empty, the values of the environment variables GEMINI_API_KEY or GOOGLE_API_KEY will be consulted, in that order. @@ -99,27 +120,7 @@ func (ga *GoogleAI) Init(ctx context.Context) []api.Action { ga.gclient = client ga.initted = true - var actions []api.Action - - models, err := listModels(googleAIProvider) - if err != nil { - panic(fmt.Errorf("GoogleAI.Init: %w", err)) - } - for n, mi := range models { - model := newModel(ga.gclient, n, mi) - actions = append(actions, model.(api.Action)) - } - - embedders, err := listEmbedders(gc.Backend) - if err != nil { - panic(fmt.Errorf("GoogleAI.Init: %w", err)) - } - for e, eOpts := range embedders { - embedder := newEmbedder(ga.gclient, e, &eOpts) - actions = append(actions, embedder.(api.Action)) - } - - return actions + return []api.Action{} } // Init initializes the VertexAI plugin and all known models and embedders. @@ -175,27 +176,7 @@ func (v *VertexAI) Init(ctx context.Context) []api.Action { v.gclient = client v.initted = true - var actions []api.Action - - models, err := listModels(vertexAIProvider) - if err != nil { - panic(fmt.Errorf("VertexAI.Init: %w", err)) - } - for n, mi := range models { - model := newModel(v.gclient, n, mi) - actions = append(actions, model.(api.Action)) - } - - embedders, err := listEmbedders(gc.Backend) - if err != nil { - panic(fmt.Errorf("VertexAI.Init: %w", err)) - } - for e, eOpts := range embedders { - embedder := newEmbedder(v.gclient, e, &eOpts) - actions = append(actions, embedder.(api.Action)) - } - - return actions + return []api.Action{} } // DefineModel defines an unknown model with the given name. @@ -316,87 +297,85 @@ func VertexAIEmbedder(g *genkit.Genkit, name string) ai.Embedder { return genkit.LookupEmbedder(g, api.NewName(vertexAIProvider, name)) } +// ListActions lists all the actions supported by the Google AI plugin. func (ga *GoogleAI) ListActions(ctx context.Context) []api.ActionDesc { - actions := []api.ActionDesc{} models, err := listGenaiModels(ctx, ga.gclient) if err != nil { return nil } + actions := []api.ActionDesc{} + + // Generative models. for _, name := range models.gemini { - metadata := map[string]any{ - "model": map[string]any{ - "supports": map[string]any{ - "media": true, - "multiturn": true, - "systemRole": true, - "tools": true, - "toolChoice": true, - "constrained": "no-tools", - }, - "versions": []string{}, - "stage": string(ai.ModelStageStable), - "customOptions": configToMap(&genai.GenerateContentConfig{}), - }, + var opts ai.ModelOptions + if knownOpts, ok := supportedGeminiModels[name]; ok { + opts = knownOpts + opts.Label = fmt.Sprintf("%s - %s", googleAILabelPrefix, opts.Label) + } else { + opts = defaultGeminiOpts + opts.Label = fmt.Sprintf("%s - %s", googleAILabelPrefix, name) } - metadata["label"] = fmt.Sprintf("%s - %s", googleAILabelPrefix, name) - actions = append(actions, api.ActionDesc{ - Type: api.ActionTypeModel, - Name: api.NewName(googleAIProvider, name), - Key: api.NewKey(api.ActionTypeModel, googleAIProvider, name), - Metadata: metadata, - }) + model := newModel(ga.gclient, name, opts) + if actionDef, ok := model.(api.Action); ok { + actions = append(actions, actionDef.Desc()) + } } + // Imagen models. for _, name := range models.imagen { - metadata := map[string]any{ - "model": map[string]any{ - "supports": map[string]any{ - "media": true, - "multiturn": true, - "systemRole": false, - "tools": false, - "toolChoice": false, - "constrained": "no-tools", - }, - "versions": []string{}, - "stage": string(ai.ModelStageStable), - "customOptions": configToMap(&genai.GenerateImagesConfig{}), - }, + var opts ai.ModelOptions + if knownOpts, ok := supportedImagenModels[name]; ok { + opts = knownOpts + opts.Label = fmt.Sprintf("%s - %s", googleAILabelPrefix, opts.Label) + } else { + opts = defaultImagenOpts + opts.Label = fmt.Sprintf("%s - %s", googleAILabelPrefix, name) } - metadata["label"] = fmt.Sprintf("%s - %s", googleAILabelPrefix, name) - actions = append(actions, api.ActionDesc{ - Type: api.ActionTypeModel, - Name: api.NewName(googleAIProvider, name), - Key: api.NewKey(api.ActionTypeModel, googleAIProvider, name), - Metadata: metadata, - }) + model := newModel(ga.gclient, name, opts) + if actionDef, ok := model.(api.Action); ok { + actions = append(actions, actionDef.Desc()) + } } + // Embedders. for _, e := range models.embedders { - actions = append(actions, api.ActionDesc{ - Type: api.ActionTypeEmbedder, - Name: api.NewName(googleAIProvider, e), - Key: api.NewKey(api.ActionTypeEmbedder, googleAIProvider, e), - }) + var embedOpts ai.EmbedderOptions + if knownOpts, ok := googleAIEmbedderConfig[e]; ok { + embedOpts = knownOpts + } else { + embedOpts = defaultEmbedOpts + embedOpts.Label = fmt.Sprintf("%s - %s", googleAILabelPrefix, e) + } + + embedder := newEmbedder(ga.gclient, e, &embedOpts) + if actionDef, ok := embedder.(api.Action); ok { + actions = append(actions, actionDef.Desc()) + } } return actions } +// ResolveAction resolves an action with the given name. func (ga *GoogleAI) ResolveAction(atype api.ActionType, name string) api.Action { - var config any switch atype { case api.ActionTypeEmbedder: return newEmbedder(ga.gclient, name, &ai.EmbedderOptions{}).(api.Action) case api.ActionTypeModel: - supports := &Multimodal - config = &genai.GenerateContentConfig{} - if strings.Contains(name, "imagen") { + var supports *ai.ModelSupports + var config any + + // TODO: Add veo case. + switch { + case strings.Contains(name, "imagen"): supports = &Media config = &genai.GenerateImagesConfig{} + default: + supports = &Multimodal + config = &genai.GenerateContentConfig{} } return newModel(ga.gclient, name, ai.ModelOptions{ @@ -410,89 +389,89 @@ func (ga *GoogleAI) ResolveAction(atype api.ActionType, name string) api.Action return nil } +// ListActions lists all the actions supported by the Vertex AI plugin. func (v *VertexAI) ListActions(ctx context.Context) []api.ActionDesc { - actions := []api.ActionDesc{} models, err := listGenaiModels(ctx, v.gclient) if err != nil { return nil } + actions := []api.ActionDesc{} + + // Gemini generative models. for _, name := range models.gemini { - metadata := map[string]any{ - "model": map[string]any{ - "supports": map[string]any{ - "media": true, - "multiturn": true, - "systemRole": true, - "tools": true, - "toolChoice": true, - "constrained": "no-tools", - }, - "versions": []string{}, - "stage": string(ai.ModelStageStable), - "customOptions": configToMap(&genai.GenerateContentConfig{}), - }, + var opts ai.ModelOptions + if knownOpts, ok := supportedGeminiModels[name]; ok { + opts = knownOpts + opts.Label = fmt.Sprintf("%s - %s", vertexAILabelPrefix, opts.Label) + } else { + opts = defaultGeminiOpts + opts.Label = fmt.Sprintf("%s - %s", vertexAILabelPrefix, name) + } + + model := newModel(v.gclient, name, opts) + if actionDef, ok := model.(api.Action); ok { + actions = append(actions, actionDef.Desc()) } - metadata["label"] = fmt.Sprintf("%s - %s", vertexAILabelPrefix, name) - actions = append(actions, api.ActionDesc{ - Type: api.ActionTypeModel, - Name: api.NewName(vertexAIProvider, name), - Key: api.NewKey(api.ActionTypeModel, vertexAIProvider, name), - Metadata: metadata, - }) } + // Imagen models. for _, name := range models.imagen { - metadata := map[string]any{ - "model": map[string]any{ - "supports": map[string]any{ - "media": true, - "multiturn": true, - "systemRole": false, - "tools": false, - "toolChoice": false, - "constrained": "no-tools", - }, - "versions": []string{}, - "stage": string(ai.ModelStageStable), - "customOptions": configToMap(&genai.GenerateImagesConfig{}), - }, + var opts ai.ModelOptions + if knownOpts, ok := supportedImagenModels[name]; ok { + opts = knownOpts + opts.Label = fmt.Sprintf("%s - %s", vertexAILabelPrefix, opts.Label) + } else { + opts = defaultImagenOpts + opts.Label = fmt.Sprintf("%s - %s", vertexAILabelPrefix, name) + } + + model := newModel(v.gclient, name, opts) + if actionDef, ok := model.(api.Action); ok { + actions = append(actions, actionDef.Desc()) } - metadata["label"] = fmt.Sprintf("%s - %s", vertexAILabelPrefix, name) - actions = append(actions, api.ActionDesc{ - Type: api.ActionTypeModel, - Name: api.NewName(vertexAIProvider, name), - Key: api.NewKey(api.ActionTypeModel, vertexAIProvider, name), - Metadata: metadata, - }) } + // Embedders. for _, e := range models.embedders { - actions = append(actions, api.ActionDesc{ - Type: api.ActionTypeEmbedder, - Name: api.NewName(vertexAIProvider, e), - Key: api.NewKey(api.ActionTypeEmbedder, vertexAIProvider, e), - }) + var embedOpts ai.EmbedderOptions + if knownOpts, ok := googleAIEmbedderConfig[e]; ok { + embedOpts = knownOpts + } else { + embedOpts = defaultEmbedOpts + embedOpts.Label = fmt.Sprintf("%s - %s", vertexAILabelPrefix, e) + } + + embedder := newEmbedder(v.gclient, e, &embedOpts) + if actionDef, ok := embedder.(api.Action); ok { + actions = append(actions, actionDef.Desc()) + } } return actions } -func (v *VertexAI) ResolveAction(atype api.ActionType, name string) api.Action { - var config any +// ResolveAction resolves an action with the given name. +func (v *VertexAI) ResolveAction(atype api.ActionType, id string) api.Action { switch atype { case api.ActionTypeEmbedder: - return newEmbedder(v.gclient, name, &ai.EmbedderOptions{}).(api.Action) + return newEmbedder(v.gclient, id, &ai.EmbedderOptions{}).(api.Action) case api.ActionTypeModel: - supports := &Multimodal - config = &genai.GenerateContentConfig{} - if strings.Contains(name, "imagen") { + var supports *ai.ModelSupports + var config any + + // TODO: Add veo case. + switch { + case strings.Contains(id, "imagen"): supports = &Media config = &genai.GenerateImagesConfig{} + default: + supports = &Multimodal + config = &genai.GenerateContentConfig{} } - return newModel(v.gclient, name, ai.ModelOptions{ - Label: fmt.Sprintf("%s - %s", vertexAILabelPrefix, name), + return newModel(v.gclient, id, ai.ModelOptions{ + Label: fmt.Sprintf("%s - %s", vertexAILabelPrefix, id), Stage: ai.ModelStageStable, Versions: []string{}, Supports: supports, diff --git a/go/plugins/googlegenai/models.go b/go/plugins/googlegenai/models.go index 9e5ca68f0b..ef43bdd7db 100644 --- a/go/plugins/googlegenai/models.go +++ b/go/plugins/googlegenai/models.go @@ -11,7 +11,6 @@ import ( "strings" "github.com/firebase/genkit/go/ai" - "github.com/firebase/genkit/go/core" "google.golang.org/genai" ) @@ -97,11 +96,6 @@ var ( imagen3Generate002, } - // Gemini models with native image support generation - imageGenModels = []string{ - gemini20FlashPrevImageGen, - } - supportedGeminiModels = map[string]ai.ModelOptions{ gemini15Flash: { Label: "Gemini 1.5 Flash", @@ -243,11 +237,6 @@ var ( }, } - googleAIEmbedders = []string{ - textembedding004, - embedding001, - } - googleAIEmbedderConfig = map[string]ai.EmbedderOptions{ textembedding004: { Dimensions: 768, @@ -255,7 +244,6 @@ var ( Supports: &ai.EmbedderSupports{ Input: []string{"text"}, }, - ConfigSchema: core.InferSchemaMap(genai.EmbedContentConfig{}), }, embedding001: { Dimensions: 768, @@ -263,7 +251,6 @@ var ( Supports: &ai.EmbedderSupports{ Input: []string{"text"}, }, - ConfigSchema: core.InferSchemaMap(genai.EmbedContentConfig{}), }, textembeddinggecko003: { Dimensions: 768, @@ -271,7 +258,6 @@ var ( Supports: &ai.EmbedderSupports{ Input: []string{"text"}, }, - ConfigSchema: core.InferSchemaMap(genai.EmbedContentConfig{}), }, textembeddinggecko002: { Dimensions: 768, @@ -279,7 +265,6 @@ var ( Supports: &ai.EmbedderSupports{ Input: []string{"text"}, }, - ConfigSchema: core.InferSchemaMap(genai.EmbedContentConfig{}), }, textembeddinggecko001: { Dimensions: 768, @@ -287,7 +272,6 @@ var ( Supports: &ai.EmbedderSupports{ Input: []string{"text"}, }, - ConfigSchema: core.InferSchemaMap(genai.EmbedContentConfig{}), }, textembeddinggeckomultilingual001: { Dimensions: 768, @@ -295,7 +279,6 @@ var ( Supports: &ai.EmbedderSupports{ Input: []string{"text"}, }, - ConfigSchema: core.InferSchemaMap(genai.EmbedContentConfig{}), }, textmultilingualembedding002: { Dimensions: 768, @@ -303,7 +286,6 @@ var ( Supports: &ai.EmbedderSupports{ Input: []string{"text"}, }, - ConfigSchema: core.InferSchemaMap(genai.EmbedContentConfig{}), }, multimodalembedding: { Dimensions: 768, @@ -315,19 +297,8 @@ var ( "video", }, }, - ConfigSchema: core.InferSchemaMap(genai.EmbedContentConfig{}), }, } - - vertexAIEmbedders = []string{ - textembeddinggecko003, - textembeddinggecko002, - textembeddinggecko001, - textembedding004, - textembeddinggeckomultilingual001, - textmultilingualembedding002, - multimodalembedding, - } ) // listModels returns a map of supported models and their capabilities @@ -371,28 +342,6 @@ func listModels(provider string) (map[string]ai.ModelOptions, error) { return models, nil } -// listEmbedders returns a list of supported embedders based on the -// detected backend -func listEmbedders(backend genai.Backend) (map[string]ai.EmbedderOptions, error) { - embeddersNames := []string{} - - switch backend { - case genai.BackendGeminiAPI: - embeddersNames = googleAIEmbedders - case genai.BackendVertexAI: - embeddersNames = vertexAIEmbedders - default: - return nil, fmt.Errorf("embedders for backend %s not found", backend) - } - - embedders := make(map[string]ai.EmbedderOptions, 0) - for _, n := range embeddersNames { - embedders[n] = googleAIEmbedderConfig[n] - } - - return embedders, nil -} - // genaiModels collects all the available models in go-genai SDK // TODO: add veo models type genaiModels struct { From cc7bbbd0579e99a2462502029312e000a3c75c83 Mon Sep 17 00:00:00 2001 From: Alex Pascal Date: Wed, 10 Sep 2025 12:21:41 -0700 Subject: [PATCH 3/4] Update json_test.go --- go/internal/base/json_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/internal/base/json_test.go b/go/internal/base/json_test.go index cb9168ac66..0acecd00d3 100644 --- a/go/internal/base/json_test.go +++ b/go/internal/base/json_test.go @@ -78,7 +78,6 @@ func TestSchemaAsMap(t *testing.T) { } want := map[string]any{ - "$id": string("https://github.com/firebase/genkit/go/internal/base/foo"), "additionalProperties": bool(false), "properties": map[string]any{ "BarField": map[string]any{ From a8b004ef3b6a8ea98276faa7f83a7872f1fe2c6f Mon Sep 17 00:00:00 2001 From: Alex Pascal Date: Wed, 10 Sep 2025 12:31:19 -0700 Subject: [PATCH 4/4] Update gemini.go --- go/plugins/googlegenai/gemini.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/plugins/googlegenai/gemini.go b/go/plugins/googlegenai/gemini.go index a131704214..8ff025135e 100644 --- a/go/plugins/googlegenai/gemini.go +++ b/go/plugins/googlegenai/gemini.go @@ -200,8 +200,8 @@ func newEmbedder(client *genai.Client, name string, embedOpts *ai.EmbedderOption var content []*genai.Content var embedConfig *genai.EmbedContentConfig - if options, _ := req.Options.(*genai.EmbedContentConfig); options != nil { - embedConfig = options + if config, ok := req.Options.(*genai.EmbedContentConfig); ok { + embedConfig = config } for _, doc := range req.Input {