diff --git a/cmd/run.go b/cmd/run.go index 3947b2f1af..4c09495330 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "strings" @@ -117,7 +118,7 @@ var runCmd = &cobra.Command{ ischedule.Start() defer ischedule.Stop() - process := process.New(name, pargs...) + process := process.NewWithContext(context.Background(), name, pargs...) res, err := process.Exec() if err != nil { if !runSilent { diff --git a/go.mod b/go.mod index 201cfdcfac..1046fe80f3 100644 --- a/go.mod +++ b/go.mod @@ -26,9 +26,9 @@ require ( github.com/yaoapp/gou v0.10.3 github.com/yaoapp/kun v0.9.0 github.com/yaoapp/xun v0.9.0 - golang.org/x/crypto v0.25.0 + golang.org/x/crypto v0.31.0 golang.org/x/net v0.27.0 - golang.org/x/text v0.16.0 + golang.org/x/text v0.21.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 rogchap.com/v8go v0.9.0 @@ -113,8 +113,8 @@ require ( golang.org/x/image v0.18.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/oauth2 v0.14.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect diff --git a/go.sum b/go.sum index 03180de741..0600be7632 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,6 @@ github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o= github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= -github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw= @@ -89,8 +87,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -173,8 +169,6 @@ github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= -github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= github.com/pkoukk/tiktoken-go v0.1.7 h1:qOBHXX4PHtvIvmOtyg1EeKlwFRiMKAcoMp4Q+bLQDmw= github.com/pkoukk/tiktoken-go v0.1.7/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -266,8 +260,8 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= @@ -297,8 +291,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -320,8 +314,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -338,8 +332,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/neo/command/command.go b/neo/command/command.go deleted file mode 100644 index 8059569fb6..0000000000 --- a/neo/command/command.go +++ /dev/null @@ -1,138 +0,0 @@ -package command - -import ( - "fmt" - "regexp" - "strings" - - "github.com/yaoapp/gou/connector" - "github.com/yaoapp/yao/aigc" - "github.com/yaoapp/yao/neo/command/driver" - "github.com/yaoapp/yao/neo/command/query" - "github.com/yaoapp/yao/openai" -) - -// DefaultStore the default store driver -var DefaultStore Store -var recmd, _ = regexp.Compile(`^\/([a-zA-Z]+) +`) -var reCmdOnly, _ = regexp.Compile(`^\/([a-zA-Z]+)$`) - -// SetStore the driver interface -func SetStore(store Store) { - DefaultStore = store -} - -// Match the command from the content -func Match(sid string, query query.Param, input string) (string, error) { - - if DefaultStore == nil { - return "", fmt.Errorf("command store is not set") - } - - // Check the command from the store - if id, cid, has := DefaultStore.GetRequest(sid); has { - fmt.Println("Match Requst:", id) - return cid, nil - } - - // Match the command use the command ID - match := reCmdOnly.FindSubmatch([]byte(strings.TrimSpace(input))) - if match == nil { - match = recmd.FindSubmatch([]byte(strings.TrimSpace(input))) - } - if match != nil { - key := fmt.Sprintf("[Index]%s", match[1]) - fmt.Println("Match Index:", key) - - if cmd, ok := DefaultStore.Get(key); ok { - fmt.Println("Match Command:", cmd.ID) - return cmd.ID, nil - } - } - - return DefaultStore.Match(query, input) -} - -// Exit the command -func Exit(sid string) error { - if DefaultStore == nil { - return fmt.Errorf("command store is not set") - } - DefaultStore.DelRequest(sid) - return nil -} - -// GetCommands get all commands -func GetCommands() ([]driver.Command, error) { - if DefaultStore == nil { - return nil, fmt.Errorf("command store is not set") - } - return DefaultStore.GetCommands() -} - -// save the command to the store -func (cmd *Command) save() error { - if DefaultStore == nil { - return nil - } - - args := []map[string]interface{}{} - for _, arg := range cmd.Args { - args = append(args, map[string]interface{}{ - "name": arg.Name, - "description": arg.Description, - "type": arg.Type, - "required": arg.Required, - }) - } - - data := driver.Command{ - ID: cmd.ID, - Name: cmd.Name, - Use: cmd.Use, - Description: cmd.Description, - Args: args, - Stack: cmd.Stack, - Path: cmd.Path, - } - - if cmd.Use != "" { - key := fmt.Sprintf("[Index]%s", cmd.Use) - err := DefaultStore.Set(key, data) - if err != nil { - return err - } - } - - return DefaultStore.Set(cmd.ID, data) -} - -// NewAI create a new AI -func (cmd *Command) newAI() (aigc.AI, error) { - - if cmd.Connector == "" || strings.HasPrefix(cmd.Connector, "moapi") { - model := "gpt-3.5-turbo" - if strings.HasPrefix(cmd.Connector, "moapi:") { - model = strings.TrimPrefix(cmd.Connector, "moapi:") - } - - ai, err := openai.NewMoapi(model) - if err != nil { - return nil, err - } - - cmd.AI = ai - return cmd.AI, nil - } - - conn, err := connector.Select(cmd.Connector) - if err != nil { - return nil, err - } - - if conn.Is(connector.OPENAI) { - return openai.New(cmd.Connector) - } - - return nil, fmt.Errorf("%s connector %s not support, should be a openai", cmd.ID, cmd.Connector) -} diff --git a/neo/command/driver/memory.go b/neo/command/driver/memory.go deleted file mode 100644 index 4b006dce12..0000000000 --- a/neo/command/driver/memory.go +++ /dev/null @@ -1,224 +0,0 @@ -package driver - -import ( - "fmt" - "strings" - "sync" - - "github.com/yaoapp/gou/connector" - "github.com/yaoapp/yao/aigc" - "github.com/yaoapp/yao/neo/command/query" - "github.com/yaoapp/yao/openai" -) - -var commands = sync.Map{} -var requests = sync.Map{} - -// Memory the memory driver -type Memory struct { - model string - ai aigc.AI - prompts []aigc.Prompt -} - -// NewMemory create a new memory driver -func NewMemory(model string, prompts []aigc.Prompt) (*Memory, error) { - - if prompts == nil || len(prompts) == 0 { - prompts = []aigc.Prompt{ - { - Role: "system", - Content: ` - - Answer my question follow this rules: - - If it can match the "name" or "description" given to you, reply the "ID" of the matched command; - - reply the "ID" only, and do not explain your answer, and do not use punctuation. - - If no matching command is found, reply me . , don't answer redundantly. - `, - }, - } - } - - mem := &Memory{model: model, prompts: prompts} - ai, err := mem.newAI() - if err != nil { - return nil, err - } - mem.ai = ai - return mem, nil -} - -// Match match the command data -func (driver *Memory) Match(query query.Param, content string) (string, error) { - - return "", fmt.Errorf("no related command found") - - // prompts := append([]aigc.Prompt{}, driver.prompts...) - // has := false - // commands.Range(func(key, value interface{}) bool { - // cmd, ok := value.(Command) - // if !ok { - // return true - // } - // if query.MatchAny(cmd.Stack, cmd.Path) { - // has = true - // bytes, err := jsoniter.Marshal(map[string]interface{}{ - // "id": cmd.ID, - // "use": cmd.Use, - // "name": cmd.Name, - // "description": cmd.Description, - // "args": cmd.Args, - // }) - // if err != nil { - // return true - // } - // prompts = append(prompts, aigc.Prompt{ - // Role: "system", - // Content: string(bytes), - // }) - // } - // return true - // }) - - // if !has { - // return "", fmt.Errorf("no related command found") - // } - - // messages := []map[string]interface{}{} - // for _, prompt := range prompts { - // messages = append(messages, map[string]interface{}{ - // "role": prompt.Role, - // "content": prompt.Content, - // }) - // } - - // messages = append(messages, map[string]interface{}{ - // "role": "user", - // "content": content, - // }) - - // res, ex := driver.ai.ChatCompletions(messages, nil, nil) - // if ex != nil { - // return "", fmt.Errorf(ex.Message) - // } - - // bytes, err := jsoniter.Marshal(res) - // if err != nil { - // return "", err - // } - - // var data struct { - // Choices []struct{ Message struct{ Content string } } - // } - // err = jsoniter.Unmarshal(bytes, &data) - // if err != nil { - // return "", err - // } - - // if len(data.Choices) == 0 { - // return "", fmt.Errorf("no related command found") - // } - - // return data.Choices[0].Message.Content, nil -} - -// Set Set the command data -func (driver *Memory) Set(key string, cmd Command) error { - commands.Store(key, cmd) - return nil -} - -// Del delete the command data -func (driver *Memory) Del(key string) { - commands.Delete(key) -} - -// Get the command data -func (driver *Memory) Get(key string) (Command, bool) { - v, ok := commands.Load(key) - if !ok { - return Command{}, false - } - cmd, ok := v.(Command) - if !ok { - return Command{}, false - } - return cmd, true -} - -// SetRequest set the command request -func (driver *Memory) SetRequest(sid, id, cid string) error { - requests.Store(sid, Request{ - ID: id, - Cid: cid, - Sid: sid, - }) - return nil -} - -// GetRequest get the command request -func (driver *Memory) GetRequest(sid string) (string, string, bool) { - v, ok := requests.Load(sid) - if !ok { - return "", "", false - } - - r, ok := v.(Request) - if !ok { - return "", "", false - } - - return r.ID, r.Cid, true -} - -// DelRequest delete the command request -func (driver *Memory) DelRequest(sid string) { - requests.Delete(sid) -} - -// GetCommands get all commands -func (driver *Memory) GetCommands() ([]Command, error) { - resulets := []Command{} - commands.Range(func(key, value interface{}) bool { - - if strings.HasPrefix(key.(string), "[Index]") { - return true - } - - cmd, ok := value.(Command) - if !ok { - return true - } - resulets = append(resulets, cmd) - return true - }) - - return resulets, nil -} - -// NewAI create a new AI -func (driver *Memory) newAI() (aigc.AI, error) { - - if driver.model == "" || strings.HasPrefix(driver.model, "moapi") { - model := "gpt-3.5-turbo" - if strings.HasPrefix(driver.model, "moapi:") { - model = strings.TrimPrefix(driver.model, "moapi:") - } - - ai, err := openai.NewMoapi(model) - if err != nil { - return nil, err - } - return ai, nil - } - - conn, err := connector.Select(driver.model) - if err != nil { - return nil, err - } - - if conn.Is(connector.OPENAI) { - return openai.New(driver.model) - } - - return nil, fmt.Errorf("connector %s not support, should be a openai", driver.model) -} diff --git a/neo/command/driver/memory_test.go b/neo/command/driver/memory_test.go deleted file mode 100644 index ff2ceb2ac4..0000000000 --- a/neo/command/driver/memory_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package driver - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/yaoapp/yao/config" - "github.com/yaoapp/yao/neo/command/query" - "github.com/yaoapp/yao/test" -) - -func TestMemorySetGetDel(t *testing.T) { - test.Prepare(t, config.Conf) - defer test.Clean() - - mem := prepare(t) - err := mem.Set("table.delete", Command{ - ID: "table.delete", - Name: "Generate test data for the table", - Description: "Generate test data for the table", - Stack: "Table.*", - Path: "*", - Args: []map[string]interface{}{ - { - "name": "data", - "type": "Array", - "description": "The data sets to generate", - "required": true, - "default": []interface{}{}, - }, - }, - }) - - if err != nil { - t.Fatal(err) - } - - cmd, has := mem.Get("table.delete") - if !has { - t.Fatal("table.delete not found") - } - - assert.Equal(t, "table.delete", cmd.ID) - mem.Del("table.delete") - - _, has = mem.Get("table.delete") - assert.False(t, has) -} - -func TestMemoryMatch(t *testing.T) { - test.Prepare(t, config.Conf) - defer test.Clean() - - mem := prepare(t) - // id, err := mem.Match(query.Param{Stack: "Table.Page.pet"}, "Generate table test data") - // if err != nil { - // t.Fatal(err) - // } - // assert.Equal(t, "table.data", id) - - _, err := mem.Match(query.Param{Stack: "Form.Page.pet", Path: "/Form/pet"}, "Generate table test data") - assert.ErrorContains(t, err, "no related command found") -} - -func prepare(t *testing.T) *Memory { - mem, err := NewMemory("gpt-3_5-turbo", nil) - if err != nil { - t.Fatal(err) - } - - mem.Set("table.data", Command{ - ID: "table.data", - Name: "Generate test data for the table", - Description: "Generate test data for the table", - Stack: "Table.*", - Path: "Table.*", - Args: []map[string]interface{}{ - { - "name": "data", - "type": "Array", - "description": "The data sets to generate", - "required": true, - "default": []interface{}{}, - }, - }, - }) - - return mem -} diff --git a/neo/command/driver/redis.go b/neo/command/driver/redis.go deleted file mode 100644 index bce7c468b3..0000000000 --- a/neo/command/driver/redis.go +++ /dev/null @@ -1 +0,0 @@ -package driver diff --git a/neo/command/driver/types.go b/neo/command/driver/types.go deleted file mode 100644 index 282dd49663..0000000000 --- a/neo/command/driver/types.go +++ /dev/null @@ -1,19 +0,0 @@ -package driver - -// Request the command request -type Request struct { - ID string - Sid string - Cid string -} - -// Command the command struct -type Command struct { - ID string `json:"-" yaml:"-"` - Use string `json:"use,omitempty"` - Name string `json:"name,omitempty"` - Description string `json:"description,omitempty"` - Args []map[string]interface{} `json:"args,omitempty"` - Stack string `json:"stack,omitempty"` - Path string `json:"path,omitempty"` -} diff --git a/neo/command/driver/weaviate.go b/neo/command/driver/weaviate.go deleted file mode 100644 index bce7c468b3..0000000000 --- a/neo/command/driver/weaviate.go +++ /dev/null @@ -1 +0,0 @@ -package driver diff --git a/neo/command/load.go b/neo/command/load.go deleted file mode 100644 index c8bf9ace14..0000000000 --- a/neo/command/load.go +++ /dev/null @@ -1,106 +0,0 @@ -package command - -import ( - "fmt" - "strings" - - "github.com/yaoapp/gou/application" - "github.com/yaoapp/yao/config" - "github.com/yaoapp/yao/share" -) - -// Commands the commands -var Commands = map[string]*Command{} - -// Autopilots the autopilots -var Autopilots = []string{} - -// Load load AIGC -func Load(cfg config.Config) error { - exts := []string{"*.cmd.yml", "*.cmd.yaml"} - messages := []string{} - - err := application.App.Walk("neo", func(root, file string, isdir bool) error { - if isdir { - return nil - } - - id := share.ID(root, file) - _, err := LoadFile(file, id) - if err != nil { - messages = append(messages, err.Error()) - } - return nil - }, exts...) - - if err != nil { - return err - } - - if len(messages) > 0 { - return fmt.Errorf("%s", strings.Join(messages, ";\n")) - } - - return nil - -} - -// LoadFile load AIGC by file -func LoadFile(file string, id string) (*Command, error) { - - data, err := application.App.Read(file) - if err != nil { - return nil, err - } - return LoadSource(data, file, id) -} - -// LoadSource load AIGC -func LoadSource(data []byte, file, id string) (*Command, error) { - - cmd := Command{ - ID: id, - Prepare: Prepare{ - Option: map[string]interface{}{}, - }, - Optional: Optional{ - Autopilot: false, - Confirm: false, - MaxAttempts: 10, - }, - } - - err := application.Parse(file, data, &cmd) - if err != nil { - return nil, err - } - - if cmd.Process == "" { - return nil, fmt.Errorf("%s process is required", id) - } - - if cmd.Prepare.Prompts == nil || len(cmd.Prepare.Prompts) == 0 { - return nil, fmt.Errorf("%s prompts is required", id) - } - - // create AI interface - cmd.AI, err = cmd.newAI() - if err != nil { - return nil, err - } - - // add to autopilots - if cmd.Optional.Autopilot { - Autopilots = append(Autopilots, id) - } - - // save - err = cmd.save() - if err != nil { - return nil, err - } - - // add to AIGCs - Commands[id] = &cmd - return Commands[id], nil -} diff --git a/neo/command/load_test.go b/neo/command/load_test.go deleted file mode 100644 index 4b012217bd..0000000000 --- a/neo/command/load_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package command - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/yaoapp/yao/config" - "github.com/yaoapp/yao/neo/command/driver" - "github.com/yaoapp/yao/test" -) - -func TestLoad(t *testing.T) { - test.Prepare(t, config.Conf) - defer test.Clean() - - Commands = map[string]*Command{} - Load(config.Conf) - check(t) -} - -func TestLoadWithStore(t *testing.T) { - test.Prepare(t, config.Conf) - defer test.Clean() - - Commands = map[string]*Command{} - mem, err := driver.NewMemory("gpt-3_5-turbo", nil) - if err != nil { - t.Fatal(err) - } - - SetStore(mem) - Load(config.Conf) - check(t) -} - -func check(t *testing.T) { - ids := map[string]bool{} - for id := range Commands { - ids[id] = true - } - assert.True(t, ids["table.data"]) - assert.GreaterOrEqual(t, len(Autopilots), 1) -} diff --git a/neo/command/prompt.go b/neo/command/prompt.go deleted file mode 100644 index a4984d98a3..0000000000 --- a/neo/command/prompt.go +++ /dev/null @@ -1,49 +0,0 @@ -package command - -import ( - jsoniter "github.com/json-iterator/go" - "github.com/yaoapp/gou/helper" - "github.com/yaoapp/kun/maps" -) - -// Replace the prompt with the context -func (prompt Prompt) Replace(data maps.MapStrAny) Prompt { - - v := map[string]interface{}{"role": prompt.Role, "content": prompt.Content} - if prompt.Name != "" { - v["name"] = prompt.Name - } - - replaced := helper.Bind(v, data) - res, ok := replaced.(map[string]interface{}) - if !ok { - return prompt - } - - if res["role"] == nil { - prompt.Role = "" - } else if role, ok := res["role"].(string); ok { - prompt.Role = role - } - - if res["name"] == nil { - prompt.Name = "" - } else if name, ok := res["name"].(string); ok { - prompt.Name = name - } - - switch content := res["content"].(type) { - case string: - prompt.Content = content - - default: - if content == nil { - prompt.Content = "" - - } else if bytes, err := jsoniter.Marshal(content); err == nil { - prompt.Content = string(bytes) - } - } - - return prompt -} diff --git a/neo/command/prompt_test.go b/neo/command/prompt_test.go deleted file mode 100644 index 4238c9ab2b..0000000000 --- a/neo/command/prompt_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package command - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/yaoapp/kun/maps" -) - -func TestPromptReplace(t *testing.T) { - - prompt := Prompt{ - Role: "{{ role }}", - Name: "{{ name }}", - Content: "{{ content }}", - } - - data := maps.Of(map[string]interface{}{ - "role": "User", - "name": "Name", - "content": "- Content\n", - }).Dot() - - prompt = prompt.Replace(data) - assert.Equal(t, "User", prompt.Role) - assert.Equal(t, "Name", prompt.Name) - assert.Equal(t, "- Content\n", prompt.Content) - - prompt = Prompt{ - Role: "Role", - Name: "{{ notfound }}", - Content: "{{ content }}", - } - - prompt = prompt.Replace(data) - assert.Equal(t, "Role", prompt.Role) - assert.Equal(t, "", prompt.Name) - assert.Equal(t, "- Content\n", prompt.Content) -} diff --git a/neo/command/query/query.go b/neo/command/query/query.go deleted file mode 100644 index 7af58b10be..0000000000 --- a/neo/command/query/query.go +++ /dev/null @@ -1,55 +0,0 @@ -package query - -import ( - "regexp" - "strings" -) - -// Param the command param -type Param struct { - Stack string `json:"stack,omitempty"` - Path string `json:"path,omitempty"` -} - -// MatchStack match the stack -func (query Param) MatchStack(stack string) bool { - - if stack == "" || stack == "*" || query.Stack == "" { - return true - } - - if stack == query.Stack { - return true - } - - matched, _ := regexp.MatchString(strings.ReplaceAll(stack, "*", ".*"), query.Stack) - return matched -} - -// MatchPath match the path -func (query Param) MatchPath(path string) bool { - if path == "" || path == "*" || query.Path == "" { - return true - } - - if path == query.Path { - return true - } - - matched, _ := regexp.MatchString(strings.ReplaceAll(path, "*", ".*"), query.Path) - return matched -} - -// MatchAny match the stack or path -func (query Param) MatchAny(stack, path string) bool { - - if path == "" || path == "-" { - return query.MatchStack(stack) - } - - if stack == "" || stack == "-" { - return query.MatchPath(path) - } - - return query.MatchStack(stack) || query.MatchPath(path) -} diff --git a/neo/command/request.go b/neo/command/request.go deleted file mode 100644 index 6e0da9ace7..0000000000 --- a/neo/command/request.go +++ /dev/null @@ -1,473 +0,0 @@ -package command - -import ( - "fmt" - "strings" - - "github.com/google/uuid" - jsoniter "github.com/json-iterator/go" - "github.com/yaoapp/gou/process" - v8 "github.com/yaoapp/gou/runtime/v8" - "github.com/yaoapp/kun/log" - "github.com/yaoapp/kun/maps" - "github.com/yaoapp/kun/utils" - "github.com/yaoapp/yao/config" - "github.com/yaoapp/yao/neo/conversation" - "github.com/yaoapp/yao/neo/message" - "rogchap.com/v8go" -) - -// Run the command -func (req *Request) Run(messages []map[string]interface{}, cb func(msg *message.JSON) int) error { - - // Enter the command mode - if input, ok := messages[len(messages)-1]["content"].(string); ok { - match := reCmdOnly.FindSubmatch([]byte(strings.TrimSpace(input))) - if match != nil { - fmt.Printf("Match Command: %s | %s\n", match[1], input) - cb(req.msg().Text("Enter the command Mode")) - cb(req.msg().Done()) - return nil - } - } - - if config.Conf.Mode == "development" { - utils.Dump("----Request Run ----") - fmt.Printf("Command Request: %s %s\n", req.Command.ID, req.sid) - fmt.Printf("Command Process: %s\n", req.Command.Process) - fmt.Printf("Command Prepare Before: %s\n", req.Command.Prepare.Before) - fmt.Printf("Command Prepare Before: %s\n", req.Command.Prepare.After) - } - - input, err := req.prepare(messages, cb) - if err != nil { - req.error(err, cb) - return err - } - - if config.Conf.Mode == "development" { - utils.Dump("----Input After Prepare ----", input) - } - - args, err := req.parseArgs(input, cb) - if err != nil { - cb(req.msg().Text("\n\n" + err.Error())) - cb(req.msg().Done()) - return nil - } - - if config.Conf.Mode == "development" { - utils.Dump("---- Command Args ----", args) - } - - // Send the command to the service - if req.Command.Optional.Confirm { - req.confirm(args, cb) - return nil - } - - // Execute the command by script - if strings.HasPrefix(req.Command.Process, "scripts.") || strings.HasPrefix(req.Command.Process, "studio.") { - - res, err := req.runScript(req.Command.Process, args, cb) - if err != nil { - cb(req.msg().Text("\n\n" + err.Error())) - return err - } - - msg := req.msg().Bind(res) - if req.Actions != nil && len(req.Actions) > 0 { - for _, action := range req.Actions { - msg.Action(action.Name, action.Type, action.Payload, action.Next) - } - } - - cb(msg.Done()) - return nil - } - - // Other process - p, err := process.Of(req.Command.Process, args...) - if err != nil { - return err - } - - res, err := p.Exec() - if err != nil { - cb(req.msg().Text("\n\n" + err.Error())) - return err - } - - msg := req.msg() - if data, ok := res.(map[string]interface{}); ok { - msg = msg.Bind(data) - } - - if req.Actions != nil && len(req.Actions) > 0 { - for _, action := range req.Actions { - msg.Action(action.Name, action.Type, action.Payload, action.Next) - } - } - - // DONE - cb(msg.Done()) - return nil -} - -// confirm the command -func (req *Request) confirm(args []interface{}, cb func(msg *message.JSON) int) { - - payload := map[string]interface{}{ - "method": "ExecCommand", - "args": []interface{}{ - req.id, - req.Command.Process, - args, - map[string]interface{}{"stack": req.ctx.Stack, "path": req.ctx.Path}, - }, - } - - msg := req.msg(). - Action("ExecCommand", "Service.__neo", payload, ""). - Confirm(). - Done() - - if req.Actions != nil && len(req.Actions) > 0 { - for _, action := range req.Actions { - msg.Action(action.Name, action.Type, action.Payload, action.Next) - } - } - - cb(msg) -} - -// validate the command -func (req *Request) parseArgs(input interface{}, cb func(msg *message.JSON) int) ([]interface{}, error) { - args := []interface{}{} - data := map[string]interface{}{} - - switch v := input.(type) { - case string: - err := jsoniter.Unmarshal([]byte(v), &data) - if err != nil { - return nil, err - } - break - - case []byte: - err := jsoniter.Unmarshal(v, &data) - if err != nil { - return nil, err - } - break - - case map[string]interface{}: - data = v - break - - default: - err := fmt.Errorf("\nInvalid input type: %T", v) - req.error(err, cb) - return nil, err - } - - // validate the args - if req.Command.Args != nil && len(req.Command.Args) > 0 { - for _, arg := range req.Command.Args { - v, ok := data[arg.Name] - if arg.Required && !ok { - err := fmt.Errorf("\nMissing required argument: %s", arg.Name) - return nil, err - } - - // @todo: validate the type - args = append(args, v) - } - } - - return args, nil -} - -// RunPrepare the command -func (req *Request) prepare(messages []map[string]interface{}, cb func(msg *message.JSON) int) (interface{}, error) { - - if config.Conf.Mode == "development" { - utils.Dump("----Messages Before Prepare ----", messages) - } - - // Before hook - data, err := req.prepareBefore(messages, cb) - if err != nil { - return nil, err - } - - // replace the pro - prompts := []Prompt{} - if req.Command.Prepare.Prompts != nil && len(req.Command.Prepare.Prompts) > 0 { - prompts = append(prompts, req.Command.Prepare.Prompts...) - } - - if data != nil { - data = maps.Of(data).Dot() - for i, prompt := range prompts { - prompts[i] = prompt.Replace(data) - } - } - - question, err := req.question(messages) - if err != nil { - req.error(err, cb) - return nil, err - } - - chatMessages, err := req.messages(prompts, question) - if err != nil { - req.error(err, cb) - return nil, err - } - - if config.Conf.Mode == "development" { - utils.Dump("----Command Prompts ----", chatMessages) - } - - // chat with AI - content := []byte{} - _, ex := req.AI.ChatCompletionsWith(req.ctx, chatMessages, req.Prepare.Option, func(data []byte) int { - msg := message.NewOpenAI(data) - if msg != nil { - if msg.IsDone() { - return 0 - } - content = msg.Append(content) - cb(req.msg().Text(msg.String())) - } - return 1 - }) - - if ex != nil { - req.error(fmt.Errorf(ex.Message), cb) - return nil, fmt.Errorf("Chat error: %s", ex.Message) - } - defer req.saveHistory(content, chatMessages) - - // After hook - args, err := req.prepareAfter(string(content), cb) - if err != nil { - log.Error("Prepare after error: %s", err.Error()) - fmt.Println(err) - return content, nil - } - - return args, nil -} - -// prepareBefore hook -func (req *Request) prepareBefore(messages []map[string]interface{}, cb func(msg *message.JSON) int) (map[string]interface{}, error) { - - if req.Prepare.Before == "" { - return nil, nil - } - - // prepare the args - args := []interface{}{ - map[string]interface{}{"stack": req.ctx.Stack, "path": req.ctx.Path}, - messages, - } - - return req.runScript(req.Prepare.Before, args, cb) -} - -// prepareAfter hook -func (req *Request) prepareAfter(content string, cb func(msg *message.JSON) int) (interface{}, error) { - - if req.Prepare.After == "" { - return content, nil - } - - // prepare the args - args := []interface{}{ - content, - map[string]interface{}{"stack": req.ctx.Stack, "path": req.ctx.Path}, // context - } - - return req.runScript(req.Prepare.After, args, cb) -} - -// saveHistory save the history -func (req *Request) saveHistory(content []byte, messages []map[string]interface{}) { - - if len(content) > 0 && req.sid != "" && len(messages) > 0 { - err := req.conversation.SaveRequest( - req.sid, - req.id, - req.Command.ID, - []map[string]interface{}{ - {"role": "user", "content": messages[len(messages)-1]["content"], "name": req.sid}, - {"role": "assistant", "content": string(content), "name": req.sid}, - }, - ) - - if err != nil { - log.Error("Save request error: %s", err.Error()) - } - } -} - -func (req *Request) error(err error, cb func(msg *message.JSON) int) { - cb(req.msg().Text(err.Error())) - cb(req.msg().Done()) - // req.Done() -} - -func (req *Request) question(messages []map[string]interface{}) (string, error) { - if len(messages) < 1 { - return "", fmt.Errorf("No messages") - } - - question, ok := messages[len(messages)-1]["content"].(string) - if !ok { - return "", fmt.Errorf("messages content is not string") - } - - return question, nil -} - -func (req *Request) messages(prompts []Prompt, question string) ([]map[string]interface{}, error) { - messages := []map[string]interface{}{} - for _, prompt := range prompts { - message := map[string]interface{}{"role": prompt.Role, "content": prompt.Content} - if prompt.Name != "" { - message["name"] = prompt.Name - } - messages = append(messages, message) - } - - history, err := req.conversation.GetRequest(req.sid, req.id) - if err != nil { - return nil, err - } - messages = append(messages, history...) - messages = append(messages, map[string]interface{}{"role": "user", "content": question, "name": req.sid}) - return messages, nil -} - -func (req *Request) runScript(id string, args []interface{}, cb func(msg *message.JSON) int) (map[string]interface{}, error) { - - namer := strings.Split(id, ".") - method := namer[len(namer)-1] - scriptID := strings.Join(namer[1:len(namer)-1], ".") - - var err error - var script *v8.Script - if namer[0] == "scripts" { - script, err = v8.Select(scriptID) - } else if namer[0] == "studio" { - script, err = v8.SelectRoot(scriptID) - } - - if err != nil { - return nil, err - } - - // make a new script context - v8ctx, err := script.NewContext(req.sid, map[string]interface{}{}) - if err != nil { - return nil, err - } - defer v8ctx.Close() - - v8ctx.WithFunction("ssWrite", func(info *v8go.FunctionCallbackInfo) *v8go.Value { - args := info.Args() - if len(args) != 1 { - return v8go.Null(info.Context().Isolate()) - } - - text := args[0].String() - cb(req.msg().Text(text)) - return v8go.Null(info.Context().Isolate()) - }) - - v8ctx.WithFunction("done", func(info *v8go.FunctionCallbackInfo) *v8go.Value { - args := info.Args() - if len(args) == 1 { - text := args[0].String() - cb(req.msg().Text(text)) - } - - cb(req.msg().Done()) - // req.Done() - return v8go.Null(v8ctx.Context.Isolate()) - }) - - res, err := v8ctx.CallWith(req.ctx, method, args...) - if err != nil { - return nil, err - } - - // return data - switch v := res.(type) { - - case bool: - return nil, nil - - case map[string]interface{}: - return v, nil - - default: - return nil, fmt.Errorf("script return type is not supported") - } - -} - -func (req *Request) msg() *message.JSON { - return message.New().Command(req.Command.Name, req.Command.ID, req.id) -} - -// NewRequest create a new request -func (cmd *Command) NewRequest(ctx Context, conversation conversation.Conversation) (*Request, error) { - - if DefaultStore == nil { - return nil, fmt.Errorf("command store is not set") - } - - if ctx.Sid == "" { - return nil, fmt.Errorf("context sid is request") - } - - // continue the request - id, cid, has := DefaultStore.GetRequest(ctx.Sid) - if has { - if cid != cmd.ID { - return nil, fmt.Errorf("request id is not match") - } - return &Request{ - Command: cmd, - sid: ctx.Sid, - id: id, - ctx: ctx, - conversation: conversation, - }, nil - } - - // create a new request - id = uuid.New().String() - err := DefaultStore.SetRequest(ctx.Sid, id, cmd.ID) - if err != nil { - return nil, err - } - - return &Request{ - Command: cmd, - sid: ctx.Sid, - id: id, - ctx: ctx, - conversation: conversation, - }, nil -} - -// Done the request done -func (req *Request) Done() { - if DefaultStore != nil { - DefaultStore.DelRequest(req.sid) - } -} diff --git a/neo/command/types.go b/neo/command/types.go deleted file mode 100644 index f9026f2870..0000000000 --- a/neo/command/types.go +++ /dev/null @@ -1,99 +0,0 @@ -package command - -import ( - "context" - - "github.com/yaoapp/yao/aigc" - "github.com/yaoapp/yao/neo/command/driver" - "github.com/yaoapp/yao/neo/command/query" - "github.com/yaoapp/yao/neo/conversation" - "github.com/yaoapp/yao/neo/message" -) - -// Request the command request -type Request struct { - id string - sid string - ctx Context - conversation conversation.Conversation - *Command -} - -// Command the command struct -type Command struct { - ID string `json:"-" yaml:"-"` - Name string `json:"name,omitempty"` - Use string `json:"use,omitempty"` - Connector string `json:"connector"` - Process string `json:"process"` - Prepare Prepare `json:"prepare"` - Description string `json:"description,omitempty"` - Optional Optional `json:"optional,omitempty"` - Args []Arg `json:"args,omitempty"` - Actions []message.Action `json:"actions,omitempty"` - Stack string `json:"stack,omitempty"` // query stack - Path string `json:"path,omitempty"` // query path - AI aigc.AI `json:"-" yaml:"-"` -} - -// Arg the argument -type Arg struct { - Name string `json:"name"` - Type string `json:"type"` - Description string `json:"description,omitempty"` - Default interface{} `json:"default,omitempty"` - Required bool `json:"required,omitempty"` -} - -// Prepare the prepare struct -type Prepare struct { - Before string `json:"before,omitempty"` - After string `json:"after,omitempty"` - Prompts []Prompt `json:"prompts"` - Option map[string]interface{} `json:"option"` -} - -// Prompt a prompt -type Prompt struct { - Role string `json:"role"` - Content string `json:"content"` - Name string `json:"name,omitempty"` -} - -// Optional optional -type Optional struct { - Autopilot bool `json:"autopilot,omitempty"` - Confirm bool `json:"confirm,omitempty"` - MaxAttempts int `json:"maxAttempts,omitempty"` // default 10 -} - -// Context the context -type Context struct { - Sid string `json:"sid" yaml:"-"` - Stack string `json:"stack,omitempty"` - Path string `json:"pathname,omitempty"` - FormData map[string]interface{} `json:"formdata,omitempty"` - Field *ContextField `json:"field,omitempty"` - Namespace string `json:"namespace,omitempty"` - Config map[string]interface{} `json:"config,omitempty"` - Signal interface{} `json:"signal,omitempty"` - context.Context `json:"-" yaml:"-"` -} - -// ContextField the context field -type ContextField struct { - Name string `json:"name,omitempty"` - Bind string `json:"bind,omitempty"` -} - -// Store the command driver -type Store interface { - Match(query query.Param, content string) (string, error) - Set(key string, cmd driver.Command) error - Get(key string) (driver.Command, bool) - Del(key string) - SetRequest(sid, id, cid string) error - GetRequest(sid string) (string, string, bool) - DelRequest(sid string) - GetCommands() ([]driver.Command, error) -} diff --git a/neo/command/context.go b/neo/context.go similarity index 70% rename from neo/command/context.go rename to neo/context.go index 5496dd1a54..ad848c95d0 100644 --- a/neo/command/context.go +++ b/neo/context.go @@ -1,4 +1,4 @@ -package command +package neo import ( "context" @@ -9,8 +9,8 @@ import ( ) // NewContext create a new context -func NewContext(sid, payload string) Context { - ctx := Context{Context: context.Background(), Sid: sid} +func NewContext(sid, cid, payload string) Context { + ctx := Context{Context: context.Background(), Sid: sid, ChatID: cid} if payload == "" { return ctx } @@ -23,14 +23,14 @@ func NewContext(sid, payload string) Context { } // NewContextWithCancel create a new context with cancel -func NewContextWithCancel(sid, payload string) (Context, context.CancelFunc) { - ctx := NewContext(sid, payload) +func NewContextWithCancel(sid, cid, payload string) (Context, context.CancelFunc) { + ctx := NewContext(sid, cid, payload) return ContextWithCancel(ctx) } // NewContextWithTimeout create a new context with timeout -func NewContextWithTimeout(sid, payload string, timeout time.Duration) (Context, context.CancelFunc) { - ctx := NewContext(sid, payload) +func NewContextWithTimeout(sid, cid, payload string, timeout time.Duration) (Context, context.CancelFunc) { + ctx := NewContext(sid, cid, payload) return ContextWithTimeout(ctx, timeout) } diff --git a/neo/conversation/mongo.go b/neo/conversation/mongo.go index 72f263f7ed..6fe46e5039 100644 --- a/neo/conversation/mongo.go +++ b/neo/conversation/mongo.go @@ -8,13 +8,23 @@ func NewMongo() *Mongo { return &Mongo{} } +// UpdateChatTitle update the chat title +func (conv *Mongo) UpdateChatTitle(sid string, cid string, title string) error { + return nil +} + +// GetChats get the chat list +func (conv *Mongo) GetChats(sid string) ([]map[string]interface{}, error) { + return []map[string]interface{}{}, nil +} + // GetHistory get the history -func (conv *Mongo) GetHistory(sid string) ([]map[string]interface{}, error) { +func (conv *Mongo) GetHistory(sid string, cid string) ([]map[string]interface{}, error) { return []map[string]interface{}{}, nil } // SaveHistory save the history -func (conv *Mongo) SaveHistory(sid string, messages []map[string]interface{}) error { +func (conv *Mongo) SaveHistory(sid string, messages []map[string]interface{}, cid string) error { return nil } diff --git a/neo/conversation/redis.go b/neo/conversation/redis.go index 9c478fc513..b19412ccf7 100644 --- a/neo/conversation/redis.go +++ b/neo/conversation/redis.go @@ -8,13 +8,23 @@ func NewRedis() *Redis { return &Redis{} } +// UpdateChatTitle update the chat title +func (conv *Redis) UpdateChatTitle(sid string, cid string, title string) error { + return nil +} + +// GetChats get the chat list +func (conv *Redis) GetChats(sid string) ([]map[string]interface{}, error) { + return []map[string]interface{}{}, nil +} + // GetHistory get the history -func (conv *Redis) GetHistory(sid string) ([]map[string]interface{}, error) { +func (conv *Redis) GetHistory(sid string, cid string) ([]map[string]interface{}, error) { return []map[string]interface{}{}, nil } // SaveHistory save the history -func (conv *Redis) SaveHistory(sid string, messages []map[string]interface{}) error { +func (conv *Redis) SaveHistory(sid string, messages []map[string]interface{}, cid string) error { return nil } diff --git a/neo/conversation/types.go b/neo/conversation/types.go index 5e3575ff1e..68ffba1794 100644 --- a/neo/conversation/types.go +++ b/neo/conversation/types.go @@ -10,8 +10,10 @@ type Setting struct { // Conversation the store interface type Conversation interface { - GetHistory(sid string) ([]map[string]interface{}, error) - SaveHistory(sid string, messages []map[string]interface{}) error + UpdateChatTitle(sid string, cid string, title string) error + GetChats(sid string) ([]map[string]interface{}, error) + GetHistory(sid string, cid string) ([]map[string]interface{}, error) + SaveHistory(sid string, messages []map[string]interface{}, cid string) error GetRequest(sid string, rid string) ([]map[string]interface{}, error) SaveRequest(sid string, rid string, cid string, messages []map[string]interface{}) error } diff --git a/neo/conversation/weaviate.go b/neo/conversation/weaviate.go index 55d3163f07..7495c44d1f 100644 --- a/neo/conversation/weaviate.go +++ b/neo/conversation/weaviate.go @@ -8,13 +8,23 @@ func NewWeaviate() *Weaviate { return &Weaviate{} } +// UpdateChatTitle update the chat title +func (conv *Weaviate) UpdateChatTitle(sid string, cid string, title string) error { + return nil +} + +// GetChats get the chat list +func (conv *Weaviate) GetChats(sid string) ([]map[string]interface{}, error) { + return []map[string]interface{}{}, nil +} + // GetHistory get the history -func (conv *Weaviate) GetHistory(sid string) ([]map[string]interface{}, error) { +func (conv *Weaviate) GetHistory(sid string, cid string) ([]map[string]interface{}, error) { return []map[string]interface{}{}, nil } // SaveHistory save the history -func (conv *Weaviate) SaveHistory(sid string, messages []map[string]interface{}) error { +func (conv *Weaviate) SaveHistory(sid string, messages []map[string]interface{}, cid string) error { return nil } diff --git a/neo/conversation/xun.go b/neo/conversation/xun.go index 6844cc694b..c34cc58b0e 100644 --- a/neo/conversation/xun.go +++ b/neo/conversation/xun.go @@ -20,11 +20,12 @@ type Xun struct { type row struct { Role string `json:"role"` - Name string `json:"name"` + Title string `json:"title"` // Chat title + Name string `json:"name"` // User name Content string `json:"content"` Sid string `json:"sid"` Rid string `json:"rid"` - Cid string `json:"cid"` + Cid string `json:"cid"` // Chat ID from chat history ExpiredAt interface{} `json:"expired_at"` } @@ -62,13 +63,49 @@ func NewXun(setting Setting) (*Xun, error) { return conv, nil } +// UpdateChatTitle update the chat title +func (conv *Xun) UpdateChatTitle(sid string, cid string, title string) error { + _, err := conv.query.Table(conv.setting.Table). + Where("sid", sid).Where("cid", cid). + Update(map[string]interface{}{"title": title}) + return err +} + +// GetChats get the chat list +func (conv *Xun) GetChats(sid string) ([]map[string]interface{}, error) { + qb := conv.query.Table(conv.setting.Table). + Select("cid"). + Where("sid", sid). + GroupBy("cid") + + if conv.setting.TTL > 0 { + qb.Where("expired_at", ">", time.Now()) + } + + res := []map[string]interface{}{} + + rows, err := qb.Get() + if err != nil { + return nil, err + } + + for _, row := range rows { + res = append(res, map[string]interface{}{ + "chat_id": row.Get("cid"), + "title": row.Get("cid"), + }) + } + + return res, nil +} + // GetHistory get the history -func (conv *Xun) GetHistory(sid string) ([]map[string]interface{}, error) { +func (conv *Xun) GetHistory(sid string, cid string) ([]map[string]interface{}, error) { qb := conv.query.Table(conv.setting.Table). Select("role", "name", "content"). Where("sid", sid). - Where("cid", ""). + Where("cid", cid). OrderBy("id", "desc") if conv.setting.TTL > 0 { @@ -98,7 +135,7 @@ func (conv *Xun) GetHistory(sid string) ([]map[string]interface{}, error) { } // SaveHistory save the history -func (conv *Xun) SaveHistory(sid string, messages []map[string]interface{}) error { +func (conv *Xun) SaveHistory(sid string, messages []map[string]interface{}, cid string) error { defer conv.clean() var expiredAt interface{} = nil @@ -113,6 +150,7 @@ func (conv *Xun) SaveHistory(sid string, messages []map[string]interface{}) erro Name: "", Content: message["content"].(string), Sid: sid, + Cid: cid, ExpiredAt: expiredAt, } @@ -214,12 +252,13 @@ func (conv *Xun) Init() error { if !has { err = conv.schema.CreateTable(conv.setting.Table, func(table schema.Blueprint) { - table.ID("id") // The ID field - table.String("sid", 255).Index() - table.String("rid", 255).Null().Index() // The request ID - table.String("cid", 200).Null().Index() // The Command ID - table.String("role", 200).Null().Index() - table.String("name", 200).Null().Index() + table.ID("id") // The ID field + table.String("sid", 255).Index() // The Session ID + table.String("rid", 255).Null().Index() // The request ID + table.String("cid", 200).Null().Index() // The Chat ID + table.String("role", 200).Null().Index() // The Message role + table.String("name", 200).Null().Index() // The User name + table.String("title", 200).Null().Index() // The Chat title table.Text("content").Null() table.TimestampTz("created_at").SetDefaultRaw("NOW()").Index() @@ -246,5 +285,15 @@ func (conv *Xun) Init() error { } } + // Auto update the title + if !tab.HasColumn("title") { + err = conv.schema.AlterTable(conv.setting.Table, func(table schema.Blueprint) { + table.String("title", 200).Null().Index() + }) + if err != nil { + return err + } + } + return nil } diff --git a/neo/conversation/xun_test.go b/neo/conversation/xun_test.go index a944607b51..e3fc069892 100644 --- a/neo/conversation/xun_test.go +++ b/neo/conversation/xun_test.go @@ -137,14 +137,15 @@ func TestXunSaveAndGetHistory(t *testing.T) { }) // save the history + cid := "123456" err = conv.SaveHistory("123456", []map[string]interface{}{ {"role": "user", "name": "user1", "content": "hello"}, {"role": "assistant", "name": "user1", "content": "Hello there, how"}, - }) + }, cid) assert.Nil(t, err) // get the history - data, err := conv.GetHistory("123456") + data, err := conv.GetHistory("123456", cid) if err != nil { t.Fatal(err) } @@ -182,3 +183,66 @@ func TestXunSaveAndGetRequest(t *testing.T) { } assert.Equal(t, 2, len(data)) } + +func TestXunSaveAndGetHistoryWithCID(t *testing.T) { + test.Prepare(t, config.Conf) + defer test.Clean() + defer capsule.Schema().DropTableIfExists("__unit_test_conversation") + + err := capsule.Schema().DropTableIfExists("__unit_test_conversation") + if err != nil { + t.Fatal(err) + } + + conv, err := NewXun(Setting{ + Connector: "default", + Table: "__unit_test_conversation", + TTL: 3600, + }) + + // save the history with specific cid + sid := "123456" + cid := "789012" + messages := []map[string]interface{}{ + {"role": "user", "name": "user1", "content": "hello"}, + {"role": "assistant", "name": "assistant1", "content": "Hi! How can I help you?"}, + } + err = conv.SaveHistory(sid, messages, cid) + assert.Nil(t, err) + + // get the history for specific cid + data, err := conv.GetHistory(sid, cid) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 2, len(data)) + + // save another message with different cid + anotherCID := "345678" + moreMessages := []map[string]interface{}{ + {"role": "user", "name": "user1", "content": "another message"}, + } + err = conv.SaveHistory(sid, moreMessages, anotherCID) + assert.Nil(t, err) + + // get history for the first cid - should still be 2 messages + data, err = conv.GetHistory(sid, cid) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 2, len(data)) + + // get history for the second cid - should be 1 message + data, err = conv.GetHistory(sid, anotherCID) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 1, len(data)) + + // get all history for the sid without specifying cid + allData, err := conv.GetHistory(sid, cid) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 2, len(allData)) +} diff --git a/neo/load.go b/neo/load.go index 90c383cafe..6773a86579 100644 --- a/neo/load.go +++ b/neo/load.go @@ -4,11 +4,8 @@ import ( "path/filepath" "github.com/yaoapp/gou/application" - "github.com/yaoapp/kun/log" "github.com/yaoapp/yao/aigc" "github.com/yaoapp/yao/config" - "github.com/yaoapp/yao/neo/command" - "github.com/yaoapp/yao/neo/command/driver" "github.com/yaoapp/yao/neo/conversation" ) @@ -23,7 +20,6 @@ func Load(cfg config.Config) error { Prompts: []aigc.Prompt{}, Option: map[string]interface{}{}, Allows: []string{}, - Command: Command{Parser: ""}, ConversationSetting: conversation.Setting{ Table: "yao_neo_conversation", Connector: "default", @@ -58,23 +54,5 @@ func Load(cfg config.Config) error { return err } - // Command Setting - parser := setting.Command.Parser - if parser == "" || parser == "default" { - parser = setting.Connector - } - - store, err := driver.NewMemory(parser, nil) - if err != nil { - return err - } - command.SetStore(store) - - // Load the commands - err = command.Load(cfg) - if err != nil { - log.Error("Command Load Error: %s", err.Error()) - } - return nil } diff --git a/neo/neo.go b/neo/neo.go index 6de9b33526..fecbff948b 100644 --- a/neo/neo.go +++ b/neo/neo.go @@ -13,8 +13,6 @@ import ( "github.com/yaoapp/gou/process" "github.com/yaoapp/kun/log" "github.com/yaoapp/yao/helper" - "github.com/yaoapp/yao/neo/command" - "github.com/yaoapp/yao/neo/command/query" "github.com/yaoapp/yao/neo/conversation" "github.com/yaoapp/yao/neo/message" "github.com/yaoapp/yao/openai" @@ -53,7 +51,7 @@ func (neo *DSL) API(router *gin.Engine, path string) error { } // set the context - ctx, cancel := command.NewContextWithCancel(sid, c.Query("context")) + ctx, cancel := NewContextWithCancel(sid, c.Query("chat_id"), c.Query("context")) defer cancel() err = neo.Answer(ctx, content, c) @@ -64,8 +62,9 @@ func (neo *DSL) API(router *gin.Engine, path string) error { }) router.GET(path, handlers...) + router.POST(path, handlers...) - // api router chat history + // api Get ChatList handlers = append(middlewares, func(c *gin.Context) { sid := c.GetString("__sid") if sid == "" { @@ -74,36 +73,19 @@ func (neo *DSL) API(router *gin.Engine, path string) error { return } - history, err := neo.Conversation.GetHistory(sid) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - c.JSON(200, map[string]interface{}{ - "data": history, - "command": nil, - }) - c.Done() - }) - router.GET(path+"/history", handlers...) - - // api router chat commands - handlers = append(middlewares, func(c *gin.Context) { - commands, err := command.GetCommands() + list, err := neo.Conversation.GetChats(sid) if err != nil { c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() return } - c.JSON(200, commands) + c.JSON(200, map[string]interface{}{"data": list}) c.Done() }) - router.GET(path+"/commands", handlers...) + router.GET(path+"/chats", handlers...) - // api router exit command mode + // api router chat history handlers = append(middlewares, func(c *gin.Context) { sid := c.GetString("__sid") if sid == "" { @@ -112,66 +94,24 @@ func (neo *DSL) API(router *gin.Engine, path string) error { return } - var payload map[string]interface{} - err := c.ShouldBindJSON(&payload) + cid := c.Query("chat_id") + history, err := neo.Conversation.GetHistory(sid, cid) if err != nil { - c.JSON(400, gin.H{"message": err.Error(), "code": 400}) - c.Done() - return - } - - cmd, ok := payload["cmd"].(string) - if !ok { - c.JSON(400, gin.H{"message": "command is required", "code": 400}) + c.JSON(500, gin.H{"message": err.Error(), "code": 500}) c.Done() return } - switch cmd { - - case "ModelList": - c.JSON(200, gin.H{"data": neo.Models, "code": 200}) - c.Done() - - case "SelectModel": - model, ok := payload["model"].(string) - if !ok { - c.JSON(400, gin.H{"message": "model is required", "code": 400}) - c.Done() - return - } - - err := neo.Select(model) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - - c.JSON(200, gin.H{"message": "success", "code": 200}) - c.Done() - - case "ExitCommandMode": - err := command.Exit(sid) - if err != nil { - c.JSON(500, gin.H{"message": err.Error(), "code": 500}) - c.Done() - return - } - c.JSON(200, gin.H{"message": "success", "code": 200}) - c.Done() - - default: - c.JSON(400, gin.H{"message": "command is not supported", "code": 400}) - } + c.JSON(200, map[string]interface{}{"data": history}) + c.Done() }) - router.POST(path, handlers...) + router.GET(path+"/history", handlers...) return nil } // Answer reply the message -func (neo *DSL) Answer(ctx command.Context, question string, c *gin.Context) error { +func (neo *DSL) Answer(ctx Context, question string, c *gin.Context) error { // get the chat messages messages, err := neo.chatMessages(ctx, question) if err != nil { @@ -190,39 +130,6 @@ func (neo *DSL) Answer(ctx command.Context, question string, c *gin.Context) err c.Header("Cache-Control", "no-cache") c.Header("Connection", "keep-alive") - // check the command - cmd, isCommand := neo.matchCommand(ctx, messages) - if isCommand { - // execute the command - req, err := cmd.NewRequest(ctx, neo.Conversation) - if err != nil { - log.Error("Command with AI error: %s", err.Error()) - done <- true - return - } - - err = req.Run(messages, func(msg *message.JSON) int { - err := neo.send(ctx, msg, messages, content, c) - if err != nil { - c.Status(500) - return 0 // break - } - - // Complete the stream - if msg.IsDone() { - return 0 // break - } - return 1 - }) - - if err != nil { - c.Status(500) - log.Error("Command with AI error: %s", err.Error()) - } - - return - } - _, ex := neo.AI.ChatCompletionsWith(ctx, messages, neo.Option, func(data []byte) int { select { @@ -266,7 +173,7 @@ func (neo *DSL) Answer(ctx command.Context, question string, c *gin.Context) err } // save the history - neo.saveHistory(ctx.Sid, content, messages) + neo.saveHistory(ctx.Sid, ctx.ChatID, content, messages) c.Status(200) // Complete the stream @@ -285,7 +192,7 @@ func (neo *DSL) Answer(ctx command.Context, question string, c *gin.Context) err } // Send send the message to the stream -func (neo *DSL) send(ctx command.Context, msg *message.JSON, messages []map[string]interface{}, content []byte, c *gin.Context) error { +func (neo *DSL) send(ctx Context, msg *message.JSON, messages []map[string]interface{}, content []byte, c *gin.Context) error { w := c.Writer @@ -356,7 +263,7 @@ func (neo *DSL) prompts() []map[string]interface{} { } // prepare the messages -func (neo *DSL) prepare(ctx command.Context, messages []map[string]interface{}) []map[string]interface{} { +func (neo *DSL) prepare(ctx Context, messages []map[string]interface{}) []map[string]interface{} { if neo.Prepare == "" { return []map[string]interface{}{} } @@ -405,9 +312,9 @@ func (neo *DSL) prepare(ctx command.Context, messages []map[string]interface{}) } // chatMessages get the chat messages -func (neo *DSL) chatMessages(ctx command.Context, content string) ([]map[string]interface{}, error) { +func (neo *DSL) chatMessages(ctx Context, content string) ([]map[string]interface{}, error) { - history, err := neo.Conversation.GetHistory(ctx.Sid) + history, err := neo.Conversation.GetHistory(ctx.Sid, ctx.ChatID) if err != nil { return nil, err } @@ -424,28 +331,8 @@ func (neo *DSL) chatMessages(ctx command.Context, content string) ([]map[string] return messages, nil } -// matchCommand match the command -func (neo *DSL) matchCommand(ctx command.Context, messages []map[string]interface{}) (*command.Command, bool) { - if len(messages) < 1 { - return nil, false - } - - input, ok := messages[len(messages)-1]["content"].(string) - if !ok { - return nil, false - } - - id, err := command.Match(ctx.Sid, query.Param{Stack: ctx.Stack, Path: ctx.Path}, input) - if err == nil && id != "" { - cmd, isCommand := command.Commands[id] - return cmd, isCommand - } - - return nil, false -} - // saveHistory save the history -func (neo *DSL) saveHistory(sid string, content []byte, messages []map[string]interface{}) { +func (neo *DSL) saveHistory(sid string, chatID string, content []byte, messages []map[string]interface{}) { if len(content) > 0 && sid != "" && len(messages) > 0 { err := neo.Conversation.SaveHistory( @@ -454,6 +341,7 @@ func (neo *DSL) saveHistory(sid string, content []byte, messages []map[string]in {"role": "user", "content": messages[len(messages)-1]["content"], "name": sid}, {"role": "assistant", "content": string(content), "name": sid}, }, + chatID, ) if err != nil { diff --git a/neo/neo_test.go b/neo/neo_test.go index d992e539f4..2292fc23e8 100644 --- a/neo/neo_test.go +++ b/neo/neo_test.go @@ -15,7 +15,6 @@ import ( httpTest "github.com/yaoapp/gou/http" "github.com/yaoapp/yao/config" "github.com/yaoapp/yao/helper" - "github.com/yaoapp/yao/neo/command" "github.com/yaoapp/yao/test" _ "github.com/yaoapp/yao/utils" ) @@ -50,7 +49,8 @@ func TestAPI(t *testing.T) { return 1 }) - assert.Contains(t, string(res), `{"done":true}`) + assert.Contains(t, string(res), `{`) + } func TestAPIAuth(t *testing.T) { @@ -111,12 +111,6 @@ func testRouter(t *testing.T) *gin.Engine { t.Fatal(err) } - // Load Commands - err = command.Load(config.Conf) - // if err != nil { - // t.Fatal(err) - // } - router := gin.New() gin.SetMode(gin.ReleaseMode) return router diff --git a/neo/types.go b/neo/types.go index f6846caaf2..32a9d011d6 100644 --- a/neo/types.go +++ b/neo/types.go @@ -1,6 +1,7 @@ package neo import ( + "context" "io" "github.com/gin-gonic/gin" @@ -21,7 +22,6 @@ type DSL struct { Write string `json:"write,omitempty"` Prompts []aigc.Prompt `json:"prompts,omitempty"` Allows []string `json:"allows,omitempty"` - Command Command `json:"command,omitempty"` Models []string `json:"models,omitempty"` AI aigc.AI `json:"-" yaml:"-"` Conversation conversation.Conversation `json:"-" yaml:"-"` @@ -35,7 +35,22 @@ type Answer interface { Header(key, value string) } -// Command setting -type Command struct { - Parser string `json:"parser,omitempty"` +// Context the context +type Context struct { + Sid string `json:"sid" yaml:"-"` + ChatID string `json:"chat_id,omitempty"` + Stack string `json:"stack,omitempty"` + Path string `json:"pathname,omitempty"` + FormData map[string]interface{} `json:"formdata,omitempty"` + Field *ContextField `json:"field,omitempty"` + Namespace string `json:"namespace,omitempty"` + Config map[string]interface{} `json:"config,omitempty"` + Signal interface{} `json:"signal,omitempty"` + context.Context `json:"-" yaml:"-"` +} + +// ContextField the context field +type ContextField struct { + Name string `json:"name,omitempty"` + Bind string `json:"bind,omitempty"` }