From b9b006f0d450fe53675a5e8aa49fc6969f47f46a Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Mon, 29 Jul 2024 16:29:07 -0400 Subject: [PATCH] remove zip files --- ethfull/convo.go | 39 +++++++++++++++++++++--------------- ethfull/generate.go | 33 ++++++++++++------------------ ethfull/state.go | 11 +++++----- injective-events/convo.go | 33 +++++++++++++++++++++--------- injective-events/generate.go | 29 ++++++++++++--------------- injective-events/state.go | 4 ++-- registry.go | 12 +++++++++++ starknet/convo.go | 33 +++++++++++++++++++++--------- starknet/generate.go | 29 ++++++++++++--------------- starknet/state.go | 4 ++-- types.go | 6 +++--- 11 files changed, 133 insertions(+), 100 deletions(-) diff --git a/ethfull/convo.go b/ethfull/convo.go index 0603950..823f4bf 100644 --- a/ethfull/convo.go +++ b/ethfull/convo.go @@ -978,24 +978,31 @@ message {{.Proto.MessageName}} {{.Proto.OutputModuleFieldName}} { ) } - c.state.sourceZip = msg.SubstreamsSourceZip - c.state.projectZip = msg.ProjectZip + c.state.sourceFiles = msg.SourceFiles + c.state.projectFiles = msg.ProjectFiles c.state.generatedCodeCompleted = true - return loop.Seq(c.msg().Messagef("Code generation complete!").Cmd(), - c.action(codegen.InputSourceDownloaded{}). - DownloadFiles(). - AddFile( - "project.zip", - msg.ProjectZip, - "application/x-zip+extract", - "\nProject files, schemas, dev environment....\n", - ).AddFile( - "substreams_src.zip", - msg.SubstreamsSourceZip, - "application/x-zip+extract", - "\nGenerated source code ready to compile.\nThis module is a Rust-based module, and you can compile it with \"make all\" in the root directory.\n", - ).Cmd()) + downloadCmd := c.action(codegen.InputSourceDownloaded{}).DownloadFiles() + + for fileName, fileContent := range msg.SourceFiles { + fileDescription := "" + if _, ok := codegen.FileDescriptions[fileName]; ok { + fileDescription = codegen.FileDescriptions[fileName] + } + + downloadCmd.AddFile(fileName, fileContent, "text/plain", fileDescription) + } + + for fileName, fileContent := range msg.ProjectFiles { + fileDescription := "" + if _, ok := codegen.FileDescriptions[fileName]; ok { + fileDescription = codegen.FileDescriptions[fileName] + } + + downloadCmd.AddFile(fileName, fileContent, "text/plain", fileDescription) + } + + return loop.Seq(c.msg().Messagef("Code generation complete!").Cmd(), downloadCmd.Cmd()) case codegen.InputSourceDownloaded: return c.NextStep() diff --git a/ethfull/generate.go b/ethfull/generate.go index 1b75fe6..38ae2d4 100644 --- a/ethfull/generate.go +++ b/ethfull/generate.go @@ -24,13 +24,13 @@ var templatesFS embed.FS func cmdGenerate(p *Project, outType outputType) loop.Cmd { return func() loop.Msg { - substreamsZip, projectZip, err := p.generate(outType) + srcFiles, projFiles, err := p.generate(outType) if err != nil { return codegen.ReturnGenerate{Err: err} } return codegen.ReturnGenerate{ - SubstreamsSourceZip: substreamsZip, - ProjectZip: projectZip, + SourceFiles: srcFiles, + ProjectFiles: projFiles, } } } @@ -69,7 +69,7 @@ func cmdBuildCompleted(content *codegen.RemoteBuildState) loop.Cmd { } } -func (p *Project) generate(outType outputType) (substreamsZip, projectZip []byte, err error) { +func (p *Project) generate(outType outputType) (srcFiles, projFiles map[string][]byte, err error) { // TODO: before doing any generation, we'll want to validate // all data points that are going into source code. // We don't want some weird things getting into `build.rs` @@ -78,25 +78,11 @@ func (p *Project) generate(outType outputType) (substreamsZip, projectZip []byte // TODO: add some checking to make sure `ParentContractName` of DynamicContract // do match a Contract that exists here. - srcFiles, projFiles, err := p.Render(outType) + srcFiles, projFiles, err = p.Render(outType) if err != nil { return nil, nil, fmt.Errorf("rendering template: %w", err) } - if len(srcFiles) != 0 { - substreamsZip, err = codegen.ZipFiles(srcFiles) - if err != nil { - return nil, nil, fmt.Errorf("zipping: %w", err) - } - } - - if len(projFiles) != 0 { - projectZip, err = codegen.ZipFiles(projFiles) - if err != nil { - return nil, nil, fmt.Errorf("zipping: %w", err) - } - } - return } @@ -135,10 +121,17 @@ func (p *Project) build(remoteBuildContentChan chan<- *codegen.RemoteBuildState) } }() + projectZip, err := codegen.ZipFiles(p.projectFiles) + if err != nil { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: err.Error(), + } + } + client := pbbuild.NewBuildServiceClient(conn) res, err := client.Build(context.Background(), &pbbuild.BuildRequest{ - SourceCode: p.sourceZip, + SourceCode: projectZip, CollectPattern: "*.spkg", Subfolder: "substreams", }, diff --git a/ethfull/state.go b/ethfull/state.go index 65e11d7..737e5de 100644 --- a/ethfull/state.go +++ b/ethfull/state.go @@ -5,14 +5,15 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/codemodus/kace" - "github.com/golang-cz/textcase" - "github.com/huandu/xstrings" "math" "regexp" "strings" "time" + "github.com/codemodus/kace" + "github.com/golang-cz/textcase" + "github.com/huandu/xstrings" + "github.com/streamingfast/eth-go" ) @@ -30,8 +31,8 @@ type Project struct { confirmDownloadOnly bool generatedCodeCompleted bool compilingBuild bool - sourceZip []byte - projectZip []byte + sourceFiles map[string][]byte + projectFiles map[string][]byte buildStarted time.Time diff --git a/injective-events/convo.go b/injective-events/convo.go index eb6e853..6dff278 100644 --- a/injective-events/convo.go +++ b/injective-events/convo.go @@ -404,18 +404,31 @@ func (c *InjectiveConvo) Update(msg loop.Msg) loop.Cmd { ) } - c.state.projectZip = msg.ProjectZip - c.state.sourceZip = msg.SubstreamsSourceZip + c.state.projectFiles = msg.ProjectFiles + c.state.sourceFiles = msg.SourceFiles + c.state.generatedCodeCompleted = true - var cmds []loop.Cmd - cmds = append(cmds, c.msg().Message("Code generation complete!").Cmd()) - cmds = append(cmds, c.action(codegen.RunBuild{}).DownloadFiles(). - AddFile("project.zip", msg.ProjectZip, "application/x-zip+extract", "\nProject files, schemas, dev environment..."). - AddFile("substreams_src.zip", msg.SubstreamsSourceZip, "application/x-zip+extract", ""). - Cmd()) + downloadCmd := c.action(codegen.InputSourceDownloaded{}).DownloadFiles() - c.state.generatedCodeCompleted = true - return loop.Seq(cmds...) + for fileName, fileContent := range msg.SourceFiles { + fileDescription := "" + if _, ok := codegen.FileDescriptions[fileName]; ok { + fileDescription = codegen.FileDescriptions[fileName] + } + + downloadCmd.AddFile(fileName, fileContent, "text/plain", fileDescription) + } + + for fileName, fileContent := range msg.ProjectFiles { + fileDescription := "" + if _, ok := codegen.FileDescriptions[fileName]; ok { + fileDescription = codegen.FileDescriptions[fileName] + } + + downloadCmd.AddFile(fileName, fileContent, "text/plain", fileDescription) + } + + return loop.Seq(c.msg().Messagef("Code generation complete!").Cmd(), downloadCmd.Cmd()) case codegen.RunBuild: return cmdBuild(c.state) diff --git a/injective-events/generate.go b/injective-events/generate.go index 964217a..d893ee4 100644 --- a/injective-events/generate.go +++ b/injective-events/generate.go @@ -119,23 +119,13 @@ func (p *Project) Render(outType outputType) (substreamsFiles map[string][]byte, return } -func (p *Project) generate(outType outputType) ([]byte, []byte, error) { +func (p *Project) generate(outType outputType) (map[string][]byte, map[string][]byte, error) { srcFiles, projectFiles, err := p.Render(outType) if err != nil { return nil, nil, fmt.Errorf("rendering template: %w", err) } - substreamsZip, err := codegen.ZipFiles(srcFiles) - if err != nil { - return nil, nil, fmt.Errorf("zipping: %w", err) - } - - projectZip, err := codegen.ZipFiles(projectFiles) - if err != nil { - return nil, nil, fmt.Errorf("zipping: %w", err) - } - - return projectZip, substreamsZip, nil + return projectFiles, srcFiles, nil } func (p *Project) build(remoteBuildContentChan chan<- *codegen.RemoteBuildState) { @@ -173,10 +163,17 @@ func (p *Project) build(remoteBuildContentChan chan<- *codegen.RemoteBuildState) } }() + projectZip, err := codegen.ZipFiles(p.projectFiles) + if err != nil { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: err.Error(), + } + } + client := pbbuild.NewBuildServiceClient(conn) res, err := client.Build(context.Background(), &pbbuild.BuildRequest{ - SourceCode: p.sourceZip, + SourceCode: projectZip, CollectPattern: "*.spkg", Subfolder: "substreams", }, @@ -229,13 +226,13 @@ func cmdGenerate(p *Project, outType outputType) loop.Cmd { p.buildStarted = time.Now() return func() loop.Msg { - projectZip, substreamsZip, err := p.generate(outType) + projectFiles, sourceFiles, err := p.generate(outType) if err != nil { return codegen.ReturnGenerate{Err: err} } return codegen.ReturnGenerate{ - ProjectZip: projectZip, - SubstreamsSourceZip: substreamsZip, + ProjectFiles: projectFiles, + SourceFiles: sourceFiles, } } } diff --git a/injective-events/state.go b/injective-events/state.go index 67d0cec..1e520de 100644 --- a/injective-events/state.go +++ b/injective-events/state.go @@ -31,8 +31,8 @@ type Project struct { generatedCodeCompleted bool compilingBuild bool - projectZip []byte - sourceZip []byte + projectFiles map[string][]byte + sourceFiles map[string][]byte buildStarted time.Time diff --git a/registry.go b/registry.go index 60a04ab..8eed1e9 100644 --- a/registry.go +++ b/registry.go @@ -33,3 +33,15 @@ func ListConversationHandlers() []*ConversationHandler { }) return handlers } + +var FileDescriptions = map[string]string{ + "contract.proto": "File containing the contract proto definition", + "build.rs": "File containing the build script for the project", + "Cargo.toml": "Cargo manifest file, a configuration file which defines the project", + "substreams.yaml": "Substreams manifest, a configuration file which defines the different modules", + "rust-toolchain.toml": "File containing the rust toolchain version", + "lib.rs": "Substreams modules definition code in Rust", + ".gitignore": "File containing the gitignore rules", + "mod.rs": "Rust module definitions file", + "contract.abi.json": "File containing the contract ABI definition", +} diff --git a/starknet/convo.go b/starknet/convo.go index ee59942..e5218dd 100644 --- a/starknet/convo.go +++ b/starknet/convo.go @@ -184,18 +184,31 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { ) } - c.state.projectZip = msg.ProjectZip - c.state.sourceZip = msg.SubstreamsSourceZip + c.state.projectFiles = msg.ProjectFiles + c.state.sourceFiles = msg.SourceFiles + c.state.generatedCodeCompleted = true - var cmds []loop.Cmd - cmds = append(cmds, c.msg().Message("Code generation complete!").Cmd()) - cmds = append(cmds, c.action(codegen.RunBuild{}).DownloadFiles(). - AddFile("project.zip", msg.ProjectZip, "application/x-zip+extract", "\nProject files, schemas, dev environment..."). - AddFile("substreams_src.zip", msg.SubstreamsSourceZip, "application/zip", ""). - Cmd()) + downloadCmd := c.action(codegen.InputSourceDownloaded{}).DownloadFiles() - c.state.generatedCodeCompleted = true - return loop.Seq(cmds...) + for fileName, fileContent := range msg.SourceFiles { + fileDescription := "" + if _, ok := codegen.FileDescriptions[fileName]; ok { + fileDescription = codegen.FileDescriptions[fileName] + } + + downloadCmd.AddFile(fileName, fileContent, "text/plain", fileDescription) + } + + for fileName, fileContent := range msg.ProjectFiles { + fileDescription := "" + if _, ok := codegen.FileDescriptions[fileName]; ok { + fileDescription = codegen.FileDescriptions[fileName] + } + + downloadCmd.AddFile(fileName, fileContent, "text/plain", fileDescription) + } + + return loop.Seq(c.msg().Messagef("Code generation complete!").Cmd(), downloadCmd.Cmd()) case codegen.RunBuild: return cmdBuild(c.state) diff --git a/starknet/generate.go b/starknet/generate.go index c64a801..65e0d9b 100644 --- a/starknet/generate.go +++ b/starknet/generate.go @@ -79,23 +79,13 @@ func (p *Project) Render(outType outputType) (substreamsFiles map[string][]byte, return } -func (p *Project) generate(outType outputType) ([]byte, []byte, error) { +func (p *Project) generate(outType outputType) (map[string][]byte, map[string][]byte, error) { srcFiles, projectFiles, err := p.Render(outType) if err != nil { return nil, nil, fmt.Errorf("rendering template: %w", err) } - substreamsZip, err := codegen.ZipFiles(srcFiles) - if err != nil { - return nil, nil, fmt.Errorf("zipping: %w", err) - } - - projectZip, err := codegen.ZipFiles(projectFiles) - if err != nil { - return nil, nil, fmt.Errorf("zipping: %w", err) - } - - return projectZip, substreamsZip, nil + return projectFiles, srcFiles, nil } func (p *Project) build(remoteBuildContentChan chan<- *codegen.RemoteBuildState) { @@ -133,10 +123,17 @@ func (p *Project) build(remoteBuildContentChan chan<- *codegen.RemoteBuildState) } }() + projectZip, err := codegen.ZipFiles(p.projectFiles) + if err != nil { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: err.Error(), + } + } + client := pbbuild.NewBuildServiceClient(conn) res, err := client.Build(context.Background(), &pbbuild.BuildRequest{ - SourceCode: p.sourceZip, + SourceCode: projectZip, CollectPattern: "*.spkg", Subfolder: "substreams", }, @@ -189,13 +186,13 @@ func cmdGenerate(p *Project, outType outputType) loop.Cmd { p.buildStarted = time.Now() return func() loop.Msg { - projectZip, substreamsZip, err := p.generate(outType) + projectFiles, sourceFiles, err := p.generate(outType) if err != nil { return codegen.ReturnGenerate{Err: err} } return codegen.ReturnGenerate{ - ProjectZip: projectZip, - SubstreamsSourceZip: substreamsZip, + ProjectFiles: projectFiles, + SourceFiles: sourceFiles, } } } diff --git a/starknet/state.go b/starknet/state.go index 5f4f98a..0dfd95b 100644 --- a/starknet/state.go +++ b/starknet/state.go @@ -39,8 +39,8 @@ type Project struct { generatedCodeCompleted bool compilingBuild bool - projectZip []byte - sourceZip []byte + projectFiles map[string][]byte + sourceFiles map[string][]byte buildStarted time.Time diff --git a/types.go b/types.go index 1c59f38..bd5cdcf 100644 --- a/types.go +++ b/types.go @@ -54,9 +54,9 @@ type InputSubgraphOutputFlavor struct{ pbconvo.UserInput_Selection } type RunGenerate struct{} type ReturnGenerate struct { - Err error - SubstreamsSourceZip []byte - ProjectZip []byte + Err error + SourceFiles map[string][]byte + ProjectFiles map[string][]byte } type RunBuild struct {