🛠️ #44 Implemented the round robin routing strategy (#71)

- Added a new round robin routing strategy
- Build routing strategy based on provided config
- Added Model & LanguageModel interfaces and refactored corresponding code to use them
- Covered the priority routing by tests
- Fixed issues in the priority routing as it was acting like round robin

docs/docs.go

@@ -318,9 +318,11 @@ const docTemplate = `{
         "routing.Strategy": {
             "type": "string",
             "enum": [
+                "round-robin",
                 "priority"
             ],
             "x-enum-varnames": [
+                "RoundRobin",
                 "Priority"
             ]
         },

docs/swagger.json

@@ -315,9 +315,11 @@
         "routing.Strategy": {
             "type": "string",
             "enum": [
+                "round-robin",
                 "priority"
             ],
             "x-enum-varnames": [
+                "RoundRobin",
                 "Priority"
             ]
         },

docs/swagger.yaml

@@ -126,9 +126,11 @@ definitions:
     type: object
   routing.Strategy:
     enum:
+    - round-robin
     - priority
     type: string
     x-enum-varnames:
+    - RoundRobin
     - Priority
   schemas.ChatMessage:
     properties:

main.go

@@ -6,19 +6,19 @@ import (
 	"glide/pkg/cmd"
 )
 
-// @title Glide Gateway
-// @version 1.0
-// @description API documentation for Glide, an open-source lightweight high-performance model gateway
+//	@title			Glide Gateway
+//	@version		1.0
+//	@description	API documentation for Glide, an open-source lightweight high-performance model gateway
 
-// @contact.name Glide Community
-// @contact.url https://github.com/modelgateway/glide
+//	@contact.name	Glide Community
+//	@contact.url	https://github.com/modelgateway/glide
 
-// @license.name Apache 2.0
-// @license.url https://github.com/modelgateway/glide/blob/develop/LICENSE
+//	@license.name	Apache 2.0
+//	@license.url	https://github.com/modelgateway/glide/blob/develop/LICENSE
 
-// @host localhost:9099
-// @BasePath /
-// @schemes http
+// @host		localhost:9099
+// @BasePath	/
+// @schemes	http
 func main() {
 	cli := cmd.NewCLI()
 

pkg/api/http/handlers.go

@@ -16,18 +16,19 @@ type Handler = func(ctx context.Context, c *app.RequestContext)
 // - https://github.com/swaggo/swag/tree/master/example/celler
 
 // LangChatHandler
-// @id glide-language-chat
-// @Summary Language Chat
-// @Description Talk to different LLMs Chat API via unified endpoint
-// @tags Language
-// @Param			router	path		string	true	"Router ID"
-// @Param			payload	body		schemas.UnifiedChatRequest	true	"Request Data"
-// @Accept json
-// @Produce json
-// @Success		200	{object}	schemas.UnifiedChatResponse
-// @Failure		400	{object}	http.ErrorSchema
-// @Failure		404	{object}	http.ErrorSchema
-// @Router /v1/language/{router}/chat [POST]
+//
+//	@id				glide-language-chat
+//	@Summary		Language Chat
+//	@Description	Talk to different LLMs Chat API via unified endpoint
+//	@tags			Language
+//	@Param			router	path	string						true	"Router ID"
+//	@Param			payload	body	schemas.UnifiedChatRequest	true	"Request Data"
+//	@Accept			json
+//	@Produce		json
+//	@Success		200	{object}	schemas.UnifiedChatResponse
+//	@Failure		400	{object}	http.ErrorSchema
+//	@Failure		404	{object}	http.ErrorSchema
+//	@Router			/v1/language/{router}/chat [POST]
 func LangChatHandler(routerManager *routers.RouterManager) Handler {
 	return func(ctx context.Context, c *app.RequestContext) {
 		var req *schemas.UnifiedChatRequest
@@ -67,14 +68,15 @@ func LangChatHandler(routerManager *routers.RouterManager) Handler {
 }
 
 // LangRoutersHandler
-// @id glide-language-routers
-// @Summary Language Router List
-// @Description Retrieve list of configured language routers and their configurations
-// @tags Language
-// @Accept json
-// @Produce json
-// @Success		200	{object}	http.RouterListSchema
-// @Router /v1/language/ [GET]
+//
+//	@id				glide-language-routers
+//	@Summary		Language Router List
+//	@Description	Retrieve list of configured language routers and their configurations
+//	@tags			Language
+//	@Accept			json
+//	@Produce		json
+//	@Success		200	{object}	http.RouterListSchema
+//	@Router			/v1/language/ [GET]
 func LangRoutersHandler(routerManager *routers.RouterManager) Handler {
 	return func(ctx context.Context, c *app.RequestContext) {
 		configuredRouters := routerManager.GetLangRouters()
@@ -89,14 +91,15 @@ func LangRoutersHandler(routerManager *routers.RouterManager) Handler {
 }
 
 // HealthHandler
-// @id glide-health
-// @Summary Gateway Health
-// @Description
-// @tags Operations
-// @Accept json
-// @Produce json
-// @Success		200	{object}	http.HealthSchema
-// @Router /v1/health/ [get]
+//
+//	@id			glide-health
+//	@Summary	Gateway Health
+//	@Description
+//	@tags		Operations
+//	@Accept		json
+//	@Produce	json
+//	@Success	200	{object}	http.HealthSchema
+//	@Router		/v1/health/ [get]
 func HealthHandler(_ context.Context, c *app.RequestContext) {
 	c.JSON(consts.StatusOK, HealthSchema{Healthy: true})
 }

pkg/providers/provider.go

@@ -16,6 +16,16 @@ type LangModelProvider interface {
 	Chat(ctx context.Context, request *schemas.UnifiedChatRequest) (*schemas.UnifiedChatResponse, error)
 }
 
+type Model interface {
+	ID() string
+	Healthy() bool
+}
+
+type LanguageModel interface {
+	Model
+	LangModelProvider
+}
+
 // LangModel
 type LangModel struct {
 	modelID     string

pkg/providers/testing.go

@@ -51,3 +51,23 @@ func (c *ProviderMock) Chat(_ context.Context, _ *schemas.UnifiedChatRequest) (*
 func (c *ProviderMock) Provider() string {
 	return "provider_mock"
 }
+
+type LangModelMock struct {
+	modelID string
+	healthy bool
+}
+
+func NewLangModelMock(ID string, healthy bool) *LangModelMock {
+	return &LangModelMock{
+		modelID: ID,
+		healthy: healthy,
+	}
+}
+
+func (m *LangModelMock) ID() string {
+	return m.modelID
+}
+
+func (m *LangModelMock) Healthy() bool {
+	return m.healthy
+}

pkg/routers/config.go

@@ -1,6 +1,8 @@
 package routers
 
 import (
+	"fmt"
+
 	"glide/pkg/providers"
 	"glide/pkg/routers/retry"
 	"glide/pkg/routers/routing"
@@ -53,10 +55,10 @@ type LangRouterConfig struct {
 }
 
 // BuildModels creates LanguageModel slice out of the given config
-func (c *LangRouterConfig) BuildModels(tel *telemetry.Telemetry) ([]*providers.LangModel, error) {
+func (c *LangRouterConfig) BuildModels(tel *telemetry.Telemetry) ([]providers.LanguageModel, error) {
 	var errs error
 
-	models := make([]*providers.LangModel, 0, len(c.Models))
+	models := make([]providers.LanguageModel, 0, len(c.Models))
 
 	for _, modelConfig := range c.Models {
 		if !modelConfig.Enabled {
@@ -102,6 +104,22 @@ func (c *LangRouterConfig) BuildRetry() *retry.ExpRetry {
 	)
 }
 
+func (c *LangRouterConfig) BuildRouting(models []providers.LanguageModel) (routing.LangModelRouting, error) {
+	m := make([]providers.Model, 0, len(models))
+	for _, model := range models {
+		m = append(m, model)
+	}
+
+	switch c.RoutingStrategy {
+	case routing.Priority:
+		return routing.NewPriorityRouting(m), nil
+	case routing.RoundRobin:
+		return routing.NewRoundRobinRouting(m), nil
+	}
+
+	return nil, fmt.Errorf("routing strategy \"%v\" is not supported, please make sure there is no typo", c.RoutingStrategy)
+}
+
 func DefaultLangRouterConfig() LangRouterConfig {
 	return LangRouterConfig{
 		Enabled:         true,

pkg/routers/router.go

@@ -24,7 +24,7 @@ type LangRouter struct {
 	Config    *LangRouterConfig
 	routing   routing.LangModelRouting
 	retry     *retry.ExpRetry
-	models    []*providers.LangModel
+	models    []providers.LanguageModel
 	telemetry *telemetry.Telemetry
 }
 
@@ -34,12 +34,17 @@ func NewLangRouter(cfg *LangRouterConfig, tel *telemetry.Telemetry) (*LangRouter
 		return nil, err
 	}
 
+	strategy, err := cfg.BuildRouting(models)
+	if err != nil {
+		return nil, err
+	}
+
 	router := &LangRouter{
 		routerID:  cfg.ID,
 		Config:    cfg,
 		models:    models,
 		retry:     cfg.BuildRetry(),
-		routing:   routing.NewPriorityRouting(models),
+		routing:   strategy,
 		telemetry: tel,
 	}
 
@@ -68,13 +73,15 @@ func (r *LangRouter) Chat(ctx context.Context, request *schemas.UnifiedChatReque
 				break
 			}
 
-			resp, err := model.Chat(ctx, request)
+			langModel := model.(providers.LanguageModel)
+
+			resp, err := langModel.Chat(ctx, request)
 			if err != nil {
 				r.telemetry.Logger.Warn(
 					"lang model failed processing chat request",
 					zap.String("routerID", r.ID()),
-					zap.String("modelID", model.ID()),
-					zap.String("provider", model.Provider()),
+					zap.String("modelID", langModel.ID()),
+					zap.String("provider", langModel.Provider()),
 					zap.Error(err),
 				)