From b72b10e99b7202de18407eb8b1910a8eb6bf7039 Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Tue, 18 Jul 2023 05:41:34 -0400 Subject: [PATCH 1/6] Add setMigration http handler --- api/http/handlerfuncs.go | 46 ++++++++++++++++++++++++++++++++++++++++ api/http/router.go | 19 ++++++++++------- 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/api/http/handlerfuncs.go b/api/http/handlerfuncs.go index 6d49afbae1..60936f2afc 100644 --- a/api/http/handlerfuncs.go +++ b/api/http/handlerfuncs.go @@ -283,6 +283,52 @@ func patchSchemaHandler(rw http.ResponseWriter, req *http.Request) { ) } +func setMigrationHandler(rw http.ResponseWriter, req *http.Request) { + cfgStr, err := readWithLimit(req.Body, rw) + if err != nil { + handleErr(req.Context(), rw, err, http.StatusInternalServerError) + return + } + + db, err := dbFromContext(req.Context()) + if err != nil { + handleErr(req.Context(), rw, err, http.StatusInternalServerError) + return + } + + txn, err := db.NewTxn(req.Context(), false) + if err != nil { + handleErr(req.Context(), rw, err, http.StatusInternalServerError) + return + } + + var cfg client.LensConfig + err = json.Unmarshal(cfgStr, &cfg) + if err != nil { + handleErr(req.Context(), rw, err, http.StatusInternalServerError) + return + } + + err = db.LensRegistry().SetMigration(req.Context(), txn, cfg) + if err != nil { + handleErr(req.Context(), rw, err, http.StatusInternalServerError) + return + } + + err = txn.Commit(req.Context()) + if err != nil { + handleErr(req.Context(), rw, err, http.StatusInternalServerError) + return + } + + sendJSON( + req.Context(), + rw, + simpleDataResponse("result", "success"), + http.StatusOK, + ) +} + func getBlockHandler(rw http.ResponseWriter, req *http.Request) { cidStr := chi.URLParam(req, "cid") diff --git a/api/http/router.go b/api/http/router.go index 6f74f59535..cdd7864c38 100644 --- a/api/http/router.go +++ b/api/http/router.go @@ -25,14 +25,15 @@ const ( Version string = "v0" versionedAPIPath string = "/api/" + Version - RootPath string = versionedAPIPath + "" - PingPath string = versionedAPIPath + "/ping" - DumpPath string = versionedAPIPath + "/debug/dump" - BlocksPath string = versionedAPIPath + "/blocks" - GraphQLPath string = versionedAPIPath + "/graphql" - SchemaPath string = versionedAPIPath + "/schema" - IndexPath string = versionedAPIPath + "/index" - PeerIDPath string = versionedAPIPath + "/peerid" + RootPath string = versionedAPIPath + "" + PingPath string = versionedAPIPath + "/ping" + DumpPath string = versionedAPIPath + "/debug/dump" + BlocksPath string = versionedAPIPath + "/blocks" + GraphQLPath string = versionedAPIPath + "/graphql" + SchemaPath string = versionedAPIPath + "/schema" + SchemaMigrationPath string = SchemaPath + "/migration" + IndexPath string = versionedAPIPath + "/index" + PeerIDPath string = versionedAPIPath + "/peerid" ) func setRoutes(h *handler) *handler { @@ -61,6 +62,8 @@ func setRoutes(h *handler) *handler { h.Get(SchemaPath, h.handle(listSchemaHandler)) h.Post(SchemaPath, h.handle(loadSchemaHandler)) h.Patch(SchemaPath, h.handle(patchSchemaHandler)) + h.Post(SchemaMigrationPath, h.handle(setMigrationHandler)) + h.Get(SchemaMigrationPath, h.handle(getMigrationHandler)) h.Post(IndexPath, h.handle(createIndexHandler)) h.Delete(IndexPath, h.handle(dropIndexHandler)) h.Get(IndexPath, h.handle(listIndexHandler)) From 29fbc2196abca4f12e80d22876068ccd905583f9 Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Tue, 18 Jul 2023 06:15:45 -0400 Subject: [PATCH 2/6] Add schema migration add cmd to CLI --- cli/cli.go | 5 + cli/schema_migration.go | 25 +++++ cli/schema_migration_set.go | 178 ++++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 cli/schema_migration.go create mode 100644 cli/schema_migration_set.go diff --git a/cli/cli.go b/cli/cli.go index c439d53015..65215680cf 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -57,6 +57,7 @@ func NewDefraCommand(cfg *config.Config) DefraCommand { rpcCmd := MakeRPCCommand(cfg) blocksCmd := MakeBlocksCommand() schemaCmd := MakeSchemaCommand() + schemaMigrationCmd := MakeSchemaMigrationCommand() indexCmd := MakeIndexCommand() clientCmd := MakeClientCommand() rpcReplicatorCmd := MakeReplicatorCommand() @@ -78,10 +79,14 @@ func NewDefraCommand(cfg *config.Config) DefraCommand { blocksCmd.AddCommand( MakeBlocksGetCommand(cfg), ) + schemaMigrationCmd.AddCommand( + MakeSchemaMigrationSetCommand(cfg), + ) schemaCmd.AddCommand( MakeSchemaAddCommand(cfg), MakeSchemaListCommand(cfg), MakeSchemaPatchCommand(cfg), + schemaMigrationCmd, ) indexCmd.AddCommand( MakeIndexCreateCommand(cfg), diff --git a/cli/schema_migration.go b/cli/schema_migration.go new file mode 100644 index 0000000000..7b37fdcabe --- /dev/null +++ b/cli/schema_migration.go @@ -0,0 +1,25 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + "github.com/spf13/cobra" +) + +func MakeSchemaMigrationCommand() *cobra.Command { + var cmd = &cobra.Command{ + Use: "migration", + Short: "Interact with the schema migration system of a running DefraDB instance", + Long: `Make set or look for existing schema migrations on a DefraDB node.`, + } + + return cmd +} diff --git a/cli/schema_migration_set.go b/cli/schema_migration_set.go new file mode 100644 index 0000000000..947e7af31e --- /dev/null +++ b/cli/schema_migration_set.go @@ -0,0 +1,178 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + + "github.com/lens-vm/lens/host-go/config/model" + "github.com/spf13/cobra" + + httpapi "github.com/sourcenetwork/defradb/api/http" + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/config" + "github.com/sourcenetwork/defradb/errors" + "github.com/sourcenetwork/defradb/logging" +) + +func MakeSchemaMigrationSetCommand(cfg *config.Config) *cobra.Command { + var lensFile string + var cmd = &cobra.Command{ + Use: "set [src] [dst] [cfg]", + Short: "Set a schema migration within DefraDB", + Long: `Set a migration between two schema versions within the local DefraDB node. + +Example: set from an argument string: + defradb client schema migration set bae123 bae456 '{"lenses": [...' + +Example: set from file: + defradb client schema migration set bae123 bae456 -f schema_migration.graphql + +Example: add from stdin: + cat schema_migration.graphql | defradb client schema migration set bae123 bae456 - + +Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, + Args: func(cmd *cobra.Command, args []string) error { + if err := cobra.MinimumNArgs(2)(cmd, args); err != nil { + return errors.New("must specify src and dst schema versions, as well as a lens cfg") + } + if err := cobra.MaximumNArgs(3)(cmd, args); err != nil { + return errors.New("must specify src and dst schema versions, as well as a lens cfg") + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) (err error) { + var lensCfgJson string + var srcSchemaVersionID string + var dstSchemaVersionID string + fi, err := os.Stdin.Stat() + if err != nil { + return err + } + + if lensFile != "" { + buf, err := os.ReadFile(lensFile) + if err != nil { + return errors.Wrap("failed to read schema file", err) + } + lensCfgJson = string(buf) + } else if len(args) == 2 { + // If the lensFile flag has not been provided then it must be provided as an arg + // and thus len(args) cannot be 2 + return errors.Wrap("must provide a lens cfg", err) + } else if isFileInfoPipe(fi) && args[2] != "-" { + log.FeedbackInfo( + cmd.Context(), + "Run 'defradb client schema migration set -' to read from stdin."+ + " Example: 'cat schema_migration.graphql | defradb client schema migration set -').", + ) + return nil + } else if args[2] == "-" { + stdin, err := readStdin() + if err != nil { + return errors.Wrap("failed to read stdin", err) + } + if len(stdin) == 0 { + return errors.New("no lens cfg in stdin provided") + } else { + lensCfgJson = stdin + } + } else { + lensCfgJson = args[2] + } + + srcSchemaVersionID = args[0] + dstSchemaVersionID = args[1] + + if lensCfgJson == "" { + return errors.New("empty lens configuration provided") + } + if srcSchemaVersionID == "" { + return errors.New("no source schema version id provided") + } + if dstSchemaVersionID == "" { + return errors.New("no destination schema version id provided") + } + + var lensCfg model.Lens + err = json.Unmarshal([]byte(lensCfgJson), &lensCfg) + if err != nil { + return errors.Wrap("invalid lens configuration", err) + } + + migrationCfg := client.LensConfig{ + SourceSchemaVersionID: srcSchemaVersionID, + DestinationSchemaVersionID: dstSchemaVersionID, + Lens: lensCfg, + } + + migrationCfgJson, err := json.Marshal(migrationCfg) + if err != nil { + return errors.Wrap("failed to marshal cfg", err) + } + + endpoint, err := httpapi.JoinPaths(cfg.API.AddressToURL(), httpapi.SchemaMigrationPath) + if err != nil { + return errors.Wrap("join paths failed", err) + } + + res, err := http.Post(endpoint.String(), "json", strings.NewReader(string(migrationCfgJson))) + if err != nil { + return errors.Wrap("failed to post schema migration", err) + } + + defer func() { + if e := res.Body.Close(); e != nil { + err = errors.Wrap(fmt.Sprintf("failed to read response body: %v", e.Error()), err) + } + }() + + response, err := io.ReadAll(res.Body) + if err != nil { + return errors.Wrap("failed to read response body", err) + } + + stdout, err := os.Stdout.Stat() + if err != nil { + return errors.Wrap("failed to stat stdout", err) + } + if isFileInfoPipe(stdout) { + cmd.Println(string(response)) + } else { + type migrationSetResponse struct { + Errors []struct { + Message string `json:"message"` + } `json:"errors"` + } + r := migrationSetResponse{} + err = json.Unmarshal(response, &r) + if err != nil { + return NewErrFailedToUnmarshalResponse(err) + } + if len(r.Errors) > 0 { + log.FeedbackError(cmd.Context(), "Failed to set schema migration", + logging.NewKV("Errors", r.Errors)) + } else { + log.FeedbackInfo(cmd.Context(), "Successfully set schema migration") + } + } + + return nil + }, + } + cmd.Flags().StringVarP(&lensFile, "file", "f", "", "Lens configuration file") + return cmd +} From 81f7b69ff90ab0f27af75b73a51838a4e1c51fed Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Tue, 18 Jul 2023 08:44:33 -0400 Subject: [PATCH 3/6] Add get migration http handler --- api/http/handlerfuncs.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/api/http/handlerfuncs.go b/api/http/handlerfuncs.go index 60936f2afc..62652eb3d6 100644 --- a/api/http/handlerfuncs.go +++ b/api/http/handlerfuncs.go @@ -329,6 +329,27 @@ func setMigrationHandler(rw http.ResponseWriter, req *http.Request) { ) } +func getMigrationHandler(rw http.ResponseWriter, req *http.Request) { + db, err := dbFromContext(req.Context()) + if err != nil { + handleErr(req.Context(), rw, err, http.StatusInternalServerError) + return + } + + cfgs := db.LensRegistry().Config() + if err != nil { + handleErr(req.Context(), rw, err, http.StatusInternalServerError) + return + } + + sendJSON( + req.Context(), + rw, + simpleDataResponse("configuration", cfgs), + http.StatusOK, + ) +} + func getBlockHandler(rw http.ResponseWriter, req *http.Request) { cidStr := chi.URLParam(req, "cid") From e6267742f8bb17296732c2962f71c18f97969c91 Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Tue, 18 Jul 2023 08:57:34 -0400 Subject: [PATCH 4/6] Add schema migration get cmd to CLI --- cli/cli.go | 1 + cli/schema_migration_get.go | 101 ++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 cli/schema_migration_get.go diff --git a/cli/cli.go b/cli/cli.go index 65215680cf..3f0ef2629b 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -81,6 +81,7 @@ func NewDefraCommand(cfg *config.Config) DefraCommand { ) schemaMigrationCmd.AddCommand( MakeSchemaMigrationSetCommand(cfg), + MakeSchemaMigrationGetCommand(cfg), ) schemaCmd.AddCommand( MakeSchemaAddCommand(cfg), diff --git a/cli/schema_migration_get.go b/cli/schema_migration_get.go new file mode 100644 index 0000000000..2fad443538 --- /dev/null +++ b/cli/schema_migration_get.go @@ -0,0 +1,101 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + + "github.com/spf13/cobra" + + httpapi "github.com/sourcenetwork/defradb/api/http" + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/config" + "github.com/sourcenetwork/defradb/errors" + "github.com/sourcenetwork/defradb/logging" +) + +func MakeSchemaMigrationGetCommand(cfg *config.Config) *cobra.Command { + var cmd = &cobra.Command{ + Use: "get", + Short: "Gets the schema migrations within DefraDB", + Long: `Gets the schema migrations within the local DefraDB node. + +Example: + defradb client schema migration get' + +Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, + Args: func(cmd *cobra.Command, args []string) error { + if err := cobra.NoArgs(cmd, args); err != nil { + return errors.New("this command take no arguments") + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) (err error) { + endpoint, err := httpapi.JoinPaths(cfg.API.AddressToURL(), httpapi.SchemaMigrationPath) + if err != nil { + return errors.Wrap("join paths failed", err) + } + + res, err := http.Get(endpoint.String()) + if err != nil { + return errors.Wrap("failed to get schema migrations", err) + } + + defer func() { + if e := res.Body.Close(); e != nil { + err = errors.Wrap(fmt.Sprintf("failed to read response body: %v", e.Error()), err) + } + }() + + response, err := io.ReadAll(res.Body) + if err != nil { + return errors.Wrap("failed to read response body", err) + } + + stdout, err := os.Stdout.Stat() + if err != nil { + return errors.Wrap("failed to stat stdout", err) + } + if isFileInfoPipe(stdout) { + cmd.Println(string(response)) + } else { + type migrationGetResponse struct { + Data struct { + Configuration []client.LensConfig `json:"configuration"` + } `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` + } + r := migrationGetResponse{} + err = json.Unmarshal(response, &r) + log.FeedbackInfo(cmd.Context(), string(response)) + if err != nil { + return NewErrFailedToUnmarshalResponse(err) + } + if len(r.Errors) > 0 { + log.FeedbackError(cmd.Context(), "Failed to get schema migrations", + logging.NewKV("Errors", r.Errors)) + } else { + log.FeedbackInfo(cmd.Context(), "Successfully got schema migrations", + logging.NewKV("Configuration", r.Data.Configuration)) + } + } + + return nil + }, + } + return cmd +} From 6855639b3a6baa39987d132115e677149eb67933 Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Tue, 18 Jul 2023 09:55:09 -0400 Subject: [PATCH 5/6] PR FIXUP - Content-type application/json --- cli/schema_migration_set.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/schema_migration_set.go b/cli/schema_migration_set.go index 947e7af31e..367b8b1079 100644 --- a/cli/schema_migration_set.go +++ b/cli/schema_migration_set.go @@ -129,7 +129,7 @@ Learn more about the DefraDB GraphQL Schema Language on https://docs.source.netw return errors.Wrap("join paths failed", err) } - res, err := http.Post(endpoint.String(), "json", strings.NewReader(string(migrationCfgJson))) + res, err := http.Post(endpoint.String(), "application/json", strings.NewReader(string(migrationCfgJson))) if err != nil { return errors.Wrap("failed to post schema migration", err) } From 22a7ea095b39d77a87b7662d7bb61c970fc162b8 Mon Sep 17 00:00:00 2001 From: Andrew Sisley Date: Tue, 18 Jul 2023 09:56:45 -0400 Subject: [PATCH 6/6] PR FIXUP - Use less odd file extension --- cli/schema_migration_set.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/schema_migration_set.go b/cli/schema_migration_set.go index 367b8b1079..d274dbce9f 100644 --- a/cli/schema_migration_set.go +++ b/cli/schema_migration_set.go @@ -39,10 +39,10 @@ Example: set from an argument string: defradb client schema migration set bae123 bae456 '{"lenses": [...' Example: set from file: - defradb client schema migration set bae123 bae456 -f schema_migration.graphql + defradb client schema migration set bae123 bae456 -f schema_migration.lens Example: add from stdin: - cat schema_migration.graphql | defradb client schema migration set bae123 bae456 - + cat schema_migration.lens | defradb client schema migration set bae123 bae456 - Learn more about the DefraDB GraphQL Schema Language on https://docs.source.network.`, Args: func(cmd *cobra.Command, args []string) error { @@ -77,7 +77,7 @@ Learn more about the DefraDB GraphQL Schema Language on https://docs.source.netw log.FeedbackInfo( cmd.Context(), "Run 'defradb client schema migration set -' to read from stdin."+ - " Example: 'cat schema_migration.graphql | defradb client schema migration set -').", + " Example: 'cat schema_migration.lens | defradb client schema migration set -').", ) return nil } else if args[2] == "-" {