diff --git a/docs/api/[Lang] Router List.bru b/docs/api/[Lang] Router List.bru new file mode 100644 index 00000000..81ccec75 --- /dev/null +++ b/docs/api/[Lang] Router List.bru @@ -0,0 +1,21 @@ +meta { + name: [Lang] Router List + type: http + seq: 3 +} + +get { + url: {{base_url}}/v1/language/ + body: json + auth: none +} + +body:json { + { + "message": { + "role": "user", + "content": "How are you doing?" + }, + "messageHistory": [] + } +} diff --git a/docs/docs.go b/docs/docs.go index 4697aea5..0a837e98 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -45,6 +45,30 @@ const docTemplate = `{ } } }, + "/v1/language/": { + "get": { + "description": "Retrieve list of configured language routers and their configurations", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Language" + ], + "summary": "Language Router List", + "operationId": "glide-language-routers", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/http.RouterListSchema" + } + } + } + } + }, "/v1/language/{router}/chat": { "post": { "description": "Talk to different LLMs Chat API via unified endpoint", @@ -117,6 +141,128 @@ const docTemplate = `{ } } }, + "http.RouterListSchema": { + "type": "object", + "properties": { + "routers": { + "type": "array", + "items": { + "$ref": "#/definitions/routers.LangRouterConfig" + } + } + } + }, + "openai.Config": { + "type": "object", + "required": [ + "apiKey" + ], + "properties": { + "apiKey": { + "type": "string" + }, + "baseUrl": { + "type": "string" + }, + "chatEndpoint": { + "type": "string" + }, + "defaultParams": { + "$ref": "#/definitions/openai.Params" + }, + "model": { + "type": "string" + } + } + }, + "openai.Params": { + "type": "object", + "properties": { + "frequency_penalty": { + "type": "integer" + }, + "logit_bias": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, + "max_tokens": { + "type": "integer" + }, + "n": { + "type": "integer" + }, + "presence_penalty": { + "type": "integer" + }, + "response_format": { + "description": "TODO: should this be a part of the chat request API?" + }, + "seed": { + "type": "integer" + }, + "stop": { + "type": "array", + "items": { + "type": "string" + } + }, + "temperature": { + "type": "number" + }, + "tool_choice": {}, + "tools": { + "type": "array", + "items": { + "type": "string" + } + }, + "top_p": { + "type": "number" + }, + "user": { + "type": "string" + } + } + }, + "providers.LangModelConfig": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "openai": { + "$ref": "#/definitions/openai.Config" + }, + "timeout": { + "type": "integer" + } + } + }, + "routers.LangRouterConfig": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "models": { + "type": "array", + "items": { + "$ref": "#/definitions/providers.LangModelConfig" + } + }, + "routers": { + "type": "string" + }, + "strategy": { + "$ref": "#/definitions/strategy.RoutingStrategy" + } + } + }, "schemas.ChatChoice": { "type": "object", "properties": { @@ -202,6 +348,15 @@ const docTemplate = `{ "type": "number" } } + }, + "strategy.RoutingStrategy": { + "type": "string", + "enum": [ + "priority" + ], + "x-enum-varnames": [ + "Priority" + ] } } }` diff --git a/docs/swagger.json b/docs/swagger.json index 95f99fef..54978778 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -42,6 +42,30 @@ } } }, + "/v1/language/": { + "get": { + "description": "Retrieve list of configured language routers and their configurations", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Language" + ], + "summary": "Language Router List", + "operationId": "glide-language-routers", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/http.RouterListSchema" + } + } + } + } + }, "/v1/language/{router}/chat": { "post": { "description": "Talk to different LLMs Chat API via unified endpoint", @@ -114,6 +138,128 @@ } } }, + "http.RouterListSchema": { + "type": "object", + "properties": { + "routers": { + "type": "array", + "items": { + "$ref": "#/definitions/routers.LangRouterConfig" + } + } + } + }, + "openai.Config": { + "type": "object", + "required": [ + "apiKey" + ], + "properties": { + "apiKey": { + "type": "string" + }, + "baseUrl": { + "type": "string" + }, + "chatEndpoint": { + "type": "string" + }, + "defaultParams": { + "$ref": "#/definitions/openai.Params" + }, + "model": { + "type": "string" + } + } + }, + "openai.Params": { + "type": "object", + "properties": { + "frequency_penalty": { + "type": "integer" + }, + "logit_bias": { + "type": "object", + "additionalProperties": { + "type": "number" + } + }, + "max_tokens": { + "type": "integer" + }, + "n": { + "type": "integer" + }, + "presence_penalty": { + "type": "integer" + }, + "response_format": { + "description": "TODO: should this be a part of the chat request API?" + }, + "seed": { + "type": "integer" + }, + "stop": { + "type": "array", + "items": { + "type": "string" + } + }, + "temperature": { + "type": "number" + }, + "tool_choice": {}, + "tools": { + "type": "array", + "items": { + "type": "string" + } + }, + "top_p": { + "type": "number" + }, + "user": { + "type": "string" + } + } + }, + "providers.LangModelConfig": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "openai": { + "$ref": "#/definitions/openai.Config" + }, + "timeout": { + "type": "integer" + } + } + }, + "routers.LangRouterConfig": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "models": { + "type": "array", + "items": { + "$ref": "#/definitions/providers.LangModelConfig" + } + }, + "routers": { + "type": "string" + }, + "strategy": { + "$ref": "#/definitions/strategy.RoutingStrategy" + } + } + }, "schemas.ChatChoice": { "type": "object", "properties": { @@ -199,6 +345,15 @@ "type": "number" } } + }, + "strategy.RoutingStrategy": { + "type": "string", + "enum": [ + "priority" + ], + "x-enum-varnames": [ + "Priority" + ] } } } \ No newline at end of file diff --git a/docs/swagger.yaml b/docs/swagger.yaml index ea630b56..8923c3cf 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -10,6 +10,86 @@ definitions: healthy: type: boolean type: object + http.RouterListSchema: + properties: + routers: + items: + $ref: '#/definitions/routers.LangRouterConfig' + type: array + type: object + openai.Config: + properties: + apiKey: + type: string + baseUrl: + type: string + chatEndpoint: + type: string + defaultParams: + $ref: '#/definitions/openai.Params' + model: + type: string + required: + - apiKey + type: object + openai.Params: + properties: + frequency_penalty: + type: integer + logit_bias: + additionalProperties: + type: number + type: object + max_tokens: + type: integer + "n": + type: integer + presence_penalty: + type: integer + response_format: + description: 'TODO: should this be a part of the chat request API?' + seed: + type: integer + stop: + items: + type: string + type: array + temperature: + type: number + tool_choice: {} + tools: + items: + type: string + type: array + top_p: + type: number + user: + type: string + type: object + providers.LangModelConfig: + properties: + enabled: + type: boolean + id: + type: string + openai: + $ref: '#/definitions/openai.Config' + timeout: + type: integer + type: object + routers.LangRouterConfig: + properties: + enabled: + type: boolean + models: + items: + $ref: '#/definitions/providers.LangModelConfig' + type: array + routers: + type: string + strategy: + $ref: '#/definitions/strategy.RoutingStrategy' + type: object schemas.ChatChoice: properties: finish_reason: @@ -70,6 +150,12 @@ definitions: total_tokens: type: number type: object + strategy.RoutingStrategy: + enum: + - priority + type: string + x-enum-varnames: + - Priority host: localhost:9099 info: contact: @@ -98,6 +184,22 @@ paths: summary: Gateway Health tags: - Operations + /v1/language/: + get: + consumes: + - application/json + description: Retrieve list of configured language routers and their configurations + operationId: glide-language-routers + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/http.RouterListSchema' + summary: Language Router List + tags: + - Language /v1/language/{router}/chat: post: consumes: diff --git a/pkg/api/http/handlers.go b/pkg/api/http/handlers.go index a310d3db..51a4e05d 100644 --- a/pkg/api/http/handlers.go +++ b/pkg/api/http/handlers.go @@ -13,7 +13,6 @@ import ( type Handler = func(ctx context.Context, c *app.RequestContext) // Swagger 101: -// // - https://github.com/swaggo/swag/tree/master/example/celler // LangChatHandler @@ -67,6 +66,28 @@ func LangChatHandler(routerManager *routers.RouterManager) Handler { } } +// LangRoutersHandler +// @id glide-language-routers +// @Summary Language Router List +// @Description Retrieve list of configured language routers and their configurations +// @tags Language +// @Accept json +// @Produce json +// @Success 200 {object} http.RouterListSchema +// @Router /v1/language/ [GET] +func LangRoutersHandler(routerManager *routers.RouterManager) Handler { + return func(ctx context.Context, c *app.RequestContext) { + configuredRouters := routerManager.GetLangRouters() + cfgs := make([]*routers.LangRouterConfig, 0, len(configuredRouters)) + + for _, router := range configuredRouters { + cfgs = append(cfgs, router.Config) + } + + c.JSON(consts.StatusOK, RouterListSchema{Routers: cfgs}) + } +} + // HealthHandler // @id glide-health // @Summary Gateway Health diff --git a/pkg/api/http/schemas.go b/pkg/api/http/schemas.go index 9a00ca01..e69bd94f 100644 --- a/pkg/api/http/schemas.go +++ b/pkg/api/http/schemas.go @@ -1,5 +1,7 @@ package http +import "glide/pkg/routers" + type ErrorSchema struct { Message string `json:"message"` } @@ -7,3 +9,7 @@ type ErrorSchema struct { type HealthSchema struct { Healthy bool `json:"healthy"` } + +type RouterListSchema struct { + Routers []*routers.LangRouterConfig `json:"routers"` +} diff --git a/pkg/api/http/server.go b/pkg/api/http/server.go index 82b6e091..fc274b91 100644 --- a/pkg/api/http/server.go +++ b/pkg/api/http/server.go @@ -37,7 +37,9 @@ func NewServer(config *ServerConfig, tel *telemetry.Telemetry, routerManager *ro func (srv *Server) Run() error { defaultGroup := srv.server.Group("/v1") + defaultGroup.GET("/language/", LangRoutersHandler(srv.routerManager)) defaultGroup.POST("/language/:router/chat/", LangChatHandler(srv.routerManager)) + defaultGroup.GET("/health/", HealthHandler) schemaDocURL := swagger.URL(fmt.Sprintf("http://%v/v1/swagger/doc.json", srv.config.HostPort)) diff --git a/pkg/config/fields/secret.go b/pkg/config/fields/secret.go index 549ba7d9..5162fc26 100644 --- a/pkg/config/fields/secret.go +++ b/pkg/config/fields/secret.go @@ -5,11 +5,15 @@ import "encoding" // Secret is a string that is marshaled in an opaque way, so we are not leaking sensitive information type Secret string -const maskedSecret = "[REDACTED]" +const maskedSecret = "***" var _ encoding.TextMarshaler = Secret("") // MarshalText marshals the secret as `[REDACTED]`. func (s Secret) MarshalText() ([]byte, error) { - return []byte(maskedSecret), nil + if len(s) == 0 { + return []byte(maskedSecret), nil + } + + return []byte(maskedSecret + s[:3]), nil } diff --git a/pkg/providers/config.go b/pkg/providers/config.go index 67f64383..d1e40c56 100644 --- a/pkg/providers/config.go +++ b/pkg/providers/config.go @@ -12,10 +12,10 @@ import ( var ErrProviderNotFound = errors.New("provider not found") type LangModelConfig struct { - ID string `yaml:"id"` - Enabled bool `yaml:"enabled"` - Timeout *time.Duration `yaml:"timeout,omitempty"` - OpenAI *openai.Config + ID string `yaml:"id" json:"id"` + Enabled bool `yaml:"enabled" json:"enabled"` + Timeout *time.Duration `yaml:"timeout,omitempty" json:"timeout" swaggertype:"primitive,integer"` + OpenAI *openai.Config `yaml:"openai" json:"openai"` // Add other providers like // Cohere *cohere.Config // Anthropic *anthropic.Config diff --git a/pkg/providers/openai/config.go b/pkg/providers/openai/config.go index 245f467f..b9398dba 100644 --- a/pkg/providers/openai/config.go +++ b/pkg/providers/openai/config.go @@ -7,19 +7,19 @@ import ( // Params defines OpenAI-specific model params with the specific validation of values // TODO: Add validations type Params struct { - Temperature float64 `yaml:"temperature,omitempty"` - TopP float64 `yaml:"top_p,omitempty"` - MaxTokens int `yaml:"max_tokens,omitempty"` - N int `yaml:"n,omitempty"` - StopWords []string `yaml:"stop,omitempty"` - FrequencyPenalty int `yaml:"frequency_penalty,omitempty"` - PresencePenalty int `yaml:"presence_penalty,omitempty"` - LogitBias *map[int]float64 `yaml:"logit_bias,omitempty"` - User *string `yaml:"user,omitempty"` - Seed *int `yaml:"seed,omitempty"` - Tools []string `yaml:"tools,omitempty"` - ToolChoice interface{} `yaml:"tool_choice,omitempty"` - ResponseFormat interface{} `yaml:"response_format,omitempty"` // TODO: should this be a part of the chat request API? + Temperature float64 `yaml:"temperature,omitempty" json:"temperature"` + TopP float64 `yaml:"top_p,omitempty" json:"top_p"` + MaxTokens int `yaml:"max_tokens,omitempty" json:"max_tokens"` + N int `yaml:"n,omitempty" json:"n"` + StopWords []string `yaml:"stop,omitempty" json:"stop"` + FrequencyPenalty int `yaml:"frequency_penalty,omitempty" json:"frequency_penalty"` + PresencePenalty int `yaml:"presence_penalty,omitempty" json:"presence_penalty"` + LogitBias *map[int]float64 `yaml:"logit_bias,omitempty" json:"logit_bias"` + User *string `yaml:"user,omitempty" json:"user"` + Seed *int `yaml:"seed,omitempty" json:"seed"` + Tools []string `yaml:"tools,omitempty" json:"tools"` + ToolChoice interface{} `yaml:"tool_choice,omitempty" json:"tool_choice"` + ResponseFormat interface{} `yaml:"response_format,omitempty" json:"response_format"` // TODO: should this be a part of the chat request API? // Stream bool `json:"stream,omitempty"` // TODO: we are not supporting this at the moment } @@ -43,11 +43,11 @@ func (p *Params) UnmarshalYAML(unmarshal func(interface{}) error) error { } type Config struct { - BaseURL string `yaml:"base_url"` - ChatEndpoint string `yaml:"chat_endpoint"` - Model string `yaml:"model"` - APIKey fields.Secret `yaml:"api_key" validate:"required"` - DefaultParams *Params `yaml:"default_params,omitempty"` + BaseURL string `yaml:"base_url" json:"baseUrl"` + ChatEndpoint string `yaml:"chat_endpoint" json:"chatEndpoint"` + Model string `yaml:"model" json:"model"` + APIKey fields.Secret `yaml:"api_key" json:"apiKey" validate:"required"` + DefaultParams *Params `yaml:"default_params,omitempty" json:"defaultParams"` } // DefaultConfig for OpenAI models diff --git a/pkg/routers/config.go b/pkg/routers/config.go index 1c4488cb..12339b6d 100644 --- a/pkg/routers/config.go +++ b/pkg/routers/config.go @@ -10,10 +10,10 @@ type Config struct { } type LangRouterConfig struct { - ID string `yaml:"id"` - Enabled bool `yaml:"enabled"` - RoutingStrategy strategy.RoutingStrategy `yaml:"strategy"` - Models []providers.LangModelConfig `yaml:"models"` + ID string `yaml:"id" json:"routers"` + Enabled bool `yaml:"enabled" json:"enabled"` + RoutingStrategy strategy.RoutingStrategy `yaml:"strategy" json:"strategy"` + Models []providers.LangModelConfig `yaml:"models" json:"models"` } func DefaultLangRouterConfig() LangRouterConfig { diff --git a/pkg/routers/manager.go b/pkg/routers/manager.go index 53519ec0..54d33fbf 100644 --- a/pkg/routers/manager.go +++ b/pkg/routers/manager.go @@ -12,7 +12,7 @@ import ( var ErrRouterNotFound = errors.New("no router found with given ID") type RouterManager struct { - config *Config + Config *Config telemetry *telemetry.Telemetry langRouterMap *map[string]*LangRouter langRouters []*LangRouter @@ -21,7 +21,7 @@ type RouterManager struct { // NewManager creates a new instance of Router Manager that creates, holds and returns all routers func NewManager(cfg *Config, tel *telemetry.Telemetry) (*RouterManager, error) { manager := RouterManager{ - config: cfg, + Config: cfg, telemetry: tel, } diff --git a/pkg/routers/router.go b/pkg/routers/router.go index 502238d9..93d71505 100644 --- a/pkg/routers/router.go +++ b/pkg/routers/router.go @@ -15,14 +15,14 @@ import ( var ErrNoModels = errors.New("no models configured for router") type LangRouter struct { - config *LangRouterConfig + Config *LangRouterConfig models []providers.LanguageModel telemetry *telemetry.Telemetry } func NewLangRouter(cfg *LangRouterConfig, tel *telemetry.Telemetry) (*LangRouter, error) { router := &LangRouter{ - config: cfg, + Config: cfg, telemetry: tel, } @@ -44,7 +44,7 @@ func (r *LangRouter) BuildModels(modelConfigs []providers.LangModelConfig) error if !modelConfig.Enabled { r.telemetry.Logger.Info( "model is disabled, skipping", - zap.String("router", r.config.ID), + zap.String("router", r.Config.ID), zap.String("model", modelConfig.ID), ) @@ -53,7 +53,7 @@ func (r *LangRouter) BuildModels(modelConfigs []providers.LangModelConfig) error r.telemetry.Logger.Debug( "init lang model", - zap.String("router", r.config.ID), + zap.String("router", r.Config.ID), zap.String("model", modelConfig.ID), )