From a34dcf488e4ef2c30458d6856768820754bce5ef Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Mon, 19 Oct 2020 16:52:43 +0300 Subject: [PATCH 1/6] Add DDL and cataloger functions configuring branch continuous export --- catalog/cataloger.go | 7 ++ catalog/cataloger_export.go | 136 +++++++++++++++++++++++++++++++ catalog/cataloger_export_test.go | 119 +++++++++++++++++++++++++++ ddl/000008_export_cfg.down.sql | 1 + ddl/000008_export_cfg.up.sql | 14 ++++ 5 files changed, 277 insertions(+) create mode 100644 catalog/cataloger_export.go create mode 100644 catalog/cataloger_export_test.go create mode 100644 ddl/000008_export_cfg.down.sql create mode 100644 ddl/000008_export_cfg.up.sql diff --git a/catalog/cataloger.go b/catalog/cataloger.go index 05a7c195d34..ca6a95b219f 100644 --- a/catalog/cataloger.go +++ b/catalog/cataloger.go @@ -152,6 +152,12 @@ type Merger interface { Merge(ctx context.Context, repository, leftBranch, rightBranch, committer, message string, metadata Metadata) (*MergeResult, error) } +type ExportConfigurator interface { + GetExportConfigurationForBranch(repository string, branch string) (ExportConfiguration, error) + GetExportConfigurations() ([]ExportConfigurationForBranch, error) + PutExportConfiguration(repository string, branch string, conf *ExportConfiguration) error +} + type Cataloger interface { RepositoryCataloger BranchCataloger @@ -160,6 +166,7 @@ type Cataloger interface { MultipartUpdateCataloger Differ Merger + ExportConfigurator io.Closer } diff --git a/catalog/cataloger_export.go b/catalog/cataloger_export.go new file mode 100644 index 00000000000..0e01cb1c865 --- /dev/null +++ b/catalog/cataloger_export.go @@ -0,0 +1,136 @@ +package catalog + +import ( + "errors" + "fmt" + "regexp" + "regexp/syntax" + "strings" + + "github.com/treeverse/lakefs/db" +) + +// ExportConfiguration describes the export configuration of a branch, as passed on wire, used +// internally, and stored in DB. +type ExportConfiguration struct { + Path string `db:"export_path" json:"exportPath"` + StatusPath string `db:"export_status_path" json:"exportStatusPath"` + LastKeysInPrefixRegexp string `db:"last_keys_in_prefix_regexp" json:"lastKeysInPrefixRegexp"` +} + +// ExportConfigurationForBranch describes how to export BranchID. It is stored in the database. +type ExportConfigurationForBranch struct { + ExportConfiguration + Repository string `db:"repository"` + Branch string `db:"branch"` +} + +func (c *cataloger) GetExportConfigurationForBranch(repository string, branch string) (ExportConfiguration, error) { + ret, err := c.db.Transact(func(tx db.Tx) (interface{}, error) { + branchID, err := c.getBranchIDCache(tx, repository, branch) + var ret ExportConfiguration + if err != nil { + return nil, err + } + err = c.db.Get(&ret, + `SELECT export_path, export_status_path, last_keys_in_prefix_regexp + FROM catalog_branches_export + WHERE branch_id = $1`, branchID) + return &ret, err + }) + if ret == nil { + return ExportConfiguration{}, err + } + return *ret.(*ExportConfiguration), err +} + +func (c *cataloger) GetExportConfigurations() ([]ExportConfigurationForBranch, error) { + ret := make([]ExportConfigurationForBranch, 0) + rows, err := c.db.Query( + `SELECT r.name repository, b.name branch, + e.export_path export_path, e.export_status_path export_status_path, + e.last_keys_in_prefix_regexp last_keys_in_prefix_regexp + FROM catalog_branches_export e JOIN catalog_branches b ON e.branch_id = b.branch_id + JOIN catalog_repositories r ON b.repository_id = r.id`) + if err != nil { + return nil, err + } + for rows.Next() { + var rec ExportConfigurationForBranch + if err = rows.Scan(&rec); err != nil { + return nil, fmt.Errorf("scan configuration %+v: %w", rows, err) + } + ret = append(ret, rec) + } + return ret, nil +} + +func (c *cataloger) PutExportConfiguration(repository string, branch string, conf *ExportConfiguration) error { + _, err := c.db.Transact(func(tx db.Tx) (interface{}, error) { + branchID, err := c.getBranchIDCache(tx, repository, branch) + if err != nil { + return nil, err + } + _, err = c.db.Exec( + `INSERT INTO catalog_branches_export ( + branch_id, export_path, export_status_path, last_keys_in_prefix_regexp) + VALUES ($1, $2, $3, $4) + ON CONFLICT (branch_id) + DO UPDATE SET (branch_id, export_path, export_status_path, last_keys_in_prefix_regexp) = + (EXCLUDED.branch_id, EXCLUDED.export_path, EXCLUDED.export_status_path, EXCLUDED.last_keys_in_prefix_regexp)`, + branchID, conf.Path, conf.StatusPath, conf.LastKeysInPrefixRegexp) + return nil, err + }) + return err +} + +const ( + groupStart = "(?:" + groupEnd = ")" +) + +// DisjunctRegexps returns a single regexp holding the disjunction ("|") of regexps. +func DisjunctRegexps(regexps []string) (*regexp.Regexp, error) { + parts := make([]string, len(regexps)) + for i, re := range regexps { + _, err := syntax.Parse(re, syntax.Perl) + if err != nil { + return nil, fmt.Errorf("%s: %w", re, err) + } + parts[i] = groupStart + re + groupEnd + } + d := strings.Join(parts, "|") + return regexp.Compile(d) +} + +var ( + ErrNotBracketed = errors.New("not a bracketed string") + unbracketRegexp = regexp.MustCompile("^" + regexp.QuoteMeta(groupStart) + "(.*)" + regexp.QuoteMeta(groupEnd) + "$") +) + +func unbracket(s string) (string, error) { + sub := unbracketRegexp.FindStringSubmatch(s) + if len(sub) == 0 { + return "", fmt.Errorf("%s: %w", s, ErrNotBracketed) + } + return sub[1], nil +} + +// DeconstructDisjunction returns the text forms of the regexps in the disjunction rex. rex +// should be constructed (only) by DisjunctRegexps. +func DeconstructDisjunction(regexp *regexp.Regexp) ([]string, error) { + s := regexp.String() + if len(s) == 0 { + return nil, nil + } + regexpParts := strings.Split(s, "|") + ret := make([]string, len(regexpParts)) + for i, regexpPart := range regexpParts { + part, err := unbracket(regexpPart) + if err != nil { + return nil, err + } + ret[i] = part + } + return ret, nil +} diff --git a/catalog/cataloger_export_test.go b/catalog/cataloger_export_test.go new file mode 100644 index 00000000000..a278b6174f1 --- /dev/null +++ b/catalog/cataloger_export_test.go @@ -0,0 +1,119 @@ +package catalog + +import ( + "context" + "errors" + syntax "regexp/syntax" + "testing" + + "github.com/go-test/deep" +) + +const ( + prefix = "prefix1" + defaultBranch = "main" + anotherBranch = "lost-not-found" +) + +func TestDisjunctRegexps(t *testing.T) { + cases := []struct { + name string + input []string + errCode syntax.ErrorCode + output string + }{ + {name: "empty", output: ""}, + {name: "one", input: []string{"^foo$"}, output: "(?:^foo$)"}, + {name: "two", input: []string{"^foo", "bar$"}, output: "(?:^foo)|(?:bar$)"}, + {name: "many", input: []string{"a", "b", "c", "z"}, output: "(?:a)|(?:b)|(?:c)|(?:z)"}, + {name: "error on first", input: []string{"("}, errCode: syntax.ErrMissingParen}, + {name: "error on second", input: []string{"a", "[z", "!"}, errCode: syntax.ErrMissingBracket}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := DisjunctRegexps(c.input) + if err != nil { + var regexpErr *syntax.Error + ok := errors.As(err, ®expErr) + if !ok { + t.Errorf("unexpected non-regexp error type %T: %s", err, err) + } else if regexpErr.Code != c.errCode { + t.Errorf("expected regexp code \"%s\" but got \"%s\"", + c.errCode, regexpErr.Code) + } + return + } + if c.errCode != "" { + t.Errorf("expected error \"%s\" but succeeded", c.errCode) + } + if got.String() != c.output { + t.Errorf("expected %s, got %s", c.output, got) + } + }) + } +} + +func TestDeconstructDisjunctionRoundtrip(t *testing.T) { + cases := []struct { + name string + input []string + }{ + {name: "empty"}, + {name: "one", input: []string{"^foo$"}}, + {name: "two", input: []string{"^foo", "bar$"}}, + {name: "many", input: []string{"a", "b", "c", "z"}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + regexp, err := DisjunctRegexps(c.input) + if err != nil { + t.Fatalf("%s: %s", c.input, err) + } + got, err := DeconstructDisjunction(regexp) + if err != nil { + t.Errorf("deconstruct %s for %v failed: %s", regexp, c.input, err) + } + if diffs := deep.Equal(c.input, got); diffs != nil { + t.Errorf("round-trip %v -> %s: %s", c.input, regexp, diffs) + } + }) + } + +} + +func TestExport_GetConfiguration(t *testing.T) { + const ( + branchID = 17 + anotherBranchID = 29 + ) + ctx := context.Background() + c := testCataloger(t) + repo := testCatalogerRepo(t, ctx, c, prefix, defaultBranch) + + cfg := ExportConfiguration{ + Path: "/path/to/export", + StatusPath: "/path/to/status", + LastKeysInPrefixRegexp: "*&@!#$", + } + + if err := c.PutExportConfiguration(repo, defaultBranch, &cfg); err != nil { + t.Fatal(err) + } + + t.Run("unconfigured branch", func(t *testing.T) { + gotCfg, err := c.GetExportConfigurationForBranch(repo, anotherBranch) + if !errors.Is(err, ErrBranchNotFound) { + t.Errorf("get configuration for unconfigured branch failed: expected ErrBranchNotFound but got %s (and %+v)", err, gotCfg) + } + }) + + t.Run("configured branch", func(t *testing.T) { + gotCfg, err := c.GetExportConfigurationForBranch(repo, defaultBranch) + if err != nil { + t.Errorf("get configuration for configured branch failed: %s", err) + } + if diffs := deep.Equal(cfg, gotCfg); diffs != nil { + t.Errorf("got other configuration than expected: %s", diffs) + } + }) +} diff --git a/ddl/000008_export_cfg.down.sql b/ddl/000008_export_cfg.down.sql new file mode 100644 index 00000000000..fa45e5c74c7 --- /dev/null +++ b/ddl/000008_export_cfg.down.sql @@ -0,0 +1 @@ +DROP TABLE catalog_branches_export; diff --git a/ddl/000008_export_cfg.up.sql b/ddl/000008_export_cfg.up.sql new file mode 100644 index 00000000000..b26e2024817 --- /dev/null +++ b/ddl/000008_export_cfg.up.sql @@ -0,0 +1,14 @@ +BEGIN; + +CREATE TABLE IF NOT EXISTS catalog_branches_export ( + branch_id integer PRIMARY KEY, + export_path varchar NOT NULL, + export_status_path varchar NOT NULL, + last_keys_in_prefix_regexp varchar NOT NULL +); + +ALTER TABLE catalog_branches_export + ADD CONSTRAINT branches_export_branches_fk + FOREIGN KEY (branch_id) REFERENCES catalog_branches(id) + ON DELETE CASCADE; +END; From 4032a01cb195b9a345f8012fc3de2542b87a79e3 Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Mon, 19 Oct 2020 16:53:46 +0300 Subject: [PATCH 2/6] Configure branch continuous export: Swagger defs and API handlers --- api/api_controller.go | 94 ++++++++++++++++++++++++++ api/api_controller_test.go | 89 +++++++++++++++++++++++++ docs/assets/js/swagger.yml | 133 ++++++++++++++++++++++++------------- swagger.yml | 85 ++++++++++++++++++++++++ 4 files changed, 353 insertions(+), 48 deletions(-) diff --git a/api/api_controller.go b/api/api_controller.go index 99f9a47a2fc..8e8a022e8ff 100644 --- a/api/api_controller.go +++ b/api/api_controller.go @@ -7,12 +7,14 @@ import ( "fmt" "net/http" "path/filepath" + "regexp" "strings" "time" "github.com/aws/aws-sdk-go/aws" "github.com/go-openapi/runtime" "github.com/go-openapi/runtime/middleware" + "github.com/go-openapi/strfmt" "github.com/go-openapi/swag" "github.com/treeverse/lakefs/api/gen/models" "github.com/treeverse/lakefs/api/gen/restapi/operations" @@ -20,6 +22,7 @@ import ( "github.com/treeverse/lakefs/api/gen/restapi/operations/branches" "github.com/treeverse/lakefs/api/gen/restapi/operations/commits" configop "github.com/treeverse/lakefs/api/gen/restapi/operations/config" + exportop "github.com/treeverse/lakefs/api/gen/restapi/operations/export" hcop "github.com/treeverse/lakefs/api/gen/restapi/operations/health_check" metadataop "github.com/treeverse/lakefs/api/gen/restapi/operations/metadata" "github.com/treeverse/lakefs/api/gen/restapi/operations/objects" @@ -184,8 +187,12 @@ func (c *Controller) Configure(api *operations.LakefsAPI) { api.RetentionGetRetentionPolicyHandler = c.RetentionGetRetentionPolicyHandler() api.RetentionUpdateRetentionPolicyHandler = c.RetentionUpdateRetentionPolicyHandler() + api.MetadataCreateSymlinkHandler = c.MetadataCreateSymlinkHandler() + api.ExportGetContinuousExportHandler = c.ExportGetContinuousExportHandler() + api.ExportSetContinuousExportHandler = c.ExportSetContinuousExportHandler() + api.ConfigGetConfigHandler = c.ConfigGetConfigHandler() } @@ -2179,6 +2186,93 @@ func (c *Controller) DetachPolicyFromGroupHandler() authop.DetachPolicyFromGroup }) } +func (c *Controller) ExportGetContinuousExportHandler() exportop.GetContinuousExportHandler { + return exportop.GetContinuousExportHandlerFunc(func(params exportop.GetContinuousExportParams, user *models.User) middleware.Responder { + deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ + { + Action: permissions.ListBranchesAction, + Resource: permissions.BranchArn(params.Repository, params.Branch), + }, + }) + if err != nil { + return exportop.NewGetContinuousExportUnauthorized(). + WithPayload(responseErrorFrom(err)) + } + + deps.LogAction("get_continuous_export") + + config, err := deps.Cataloger.GetExportConfigurationForBranch(params.Repository, params.Branch) + if errors.Is(err, catalog.ErrRepositoryNotFound) || errors.Is(err, catalog.ErrBranchNotFound) { + return exportop.NewGetContinuousExportNotFound(). + WithPayload(responseErrorFrom(err)) + } + if err != nil { + return exportop.NewGetContinuousExportDefault(http.StatusInternalServerError). + WithPayload(responseErrorFrom(err)) + } + + prefixRegexp, err := regexp.Compile(config.LastKeysInPrefixRegexp) + if err != nil { + return exportop.NewGetContinuousExportDefault(http.StatusInternalServerError). + WithPayload(responseErrorFrom(err)) + } + + prefixRegexps, err := catalog.DeconstructDisjunction(prefixRegexp) + if err != nil { + return exportop.NewGetContinuousExportDefault(http.StatusInternalServerError). + WithPayload(responseErrorFrom(fmt.Errorf( + "cannot deconstruct last_keys_in_prefix_regexp %s: %w", config.LastKeysInPrefixRegexp, err))) + } + + payload := models.ContinuousExportConfiguration{ + ExportPath: strfmt.URI(config.Path), + ExportStatusPath: strfmt.URI(config.StatusPath), + LastKeysInPrefixRegexp: prefixRegexps, + } + return exportop.NewGetContinuousExportOK().WithPayload(&payload) + + }) +} + +func (c *Controller) ExportSetContinuousExportHandler() exportop.SetContinuousExportHandlerFunc { + return exportop.SetContinuousExportHandlerFunc(func(params exportop.SetContinuousExportParams, user *models.User) middleware.Responder { + deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ + { + Action: permissions.CreateBranchAction, + Resource: permissions.BranchArn(params.Repository, params.Branch), + }, + }) + if err != nil { + return exportop.NewSetContinuousExportUnauthorized(). + WithPayload(responseErrorFrom(err)) + } + + deps.LogAction("set_continuous_export") + + lastKeysInPrefixRegexp, err := catalog.DisjunctRegexps(params.Config.LastKeysInPrefixRegexp) + if err != nil { + return exportop.NewSetContinuousExportDefault(http.StatusInternalServerError). + WithPayload(responseError("join last keys in prefix regexps: %s", err)) + } + config := catalog.ExportConfiguration{ + Path: params.Config.ExportPath.String(), + StatusPath: params.Config.ExportStatusPath.String(), + LastKeysInPrefixRegexp: lastKeysInPrefixRegexp.String(), + } + err = deps.Cataloger.PutExportConfiguration(params.Repository, params.Branch, &config) + if errors.Is(err, catalog.ErrRepositoryNotFound) || errors.Is(err, catalog.ErrBranchNotFound) { + return exportop.NewSetContinuousExportNotFound(). + WithPayload(responseErrorFrom(err)) + } + if err != nil { + return exportop.NewSetContinuousExportDefault(http.StatusInternalServerError). + WithPayload(responseErrorFrom(err)) + } + + return exportop.NewSetContinuousExportCreated() + }) +} + func (c *Controller) RetentionGetRetentionPolicyHandler() retentionop.GetRetentionPolicyHandler { return retentionop.GetRetentionPolicyHandlerFunc(func(params retentionop.GetRetentionPolicyParams, user *models.User) middleware.Responder { deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ diff --git a/api/api_controller_test.go b/api/api_controller_test.go index 089cf0233f7..78d27971606 100644 --- a/api/api_controller_test.go +++ b/api/api_controller_test.go @@ -14,6 +14,7 @@ import ( "github.com/go-openapi/runtime" httptransport "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" "github.com/go-openapi/swag" "github.com/go-test/deep" "github.com/treeverse/lakefs/api/gen/client" @@ -21,6 +22,7 @@ import ( "github.com/treeverse/lakefs/api/gen/client/branches" "github.com/treeverse/lakefs/api/gen/client/commits" "github.com/treeverse/lakefs/api/gen/client/config" + "github.com/treeverse/lakefs/api/gen/client/export" "github.com/treeverse/lakefs/api/gen/client/objects" "github.com/treeverse/lakefs/api/gen/client/repositories" "github.com/treeverse/lakefs/api/gen/client/retention" @@ -1354,6 +1356,93 @@ func TestHandler_ConfigHandlers(t *testing.T) { }) } +func TestHandler_ContinuousExportHandlers(t *testing.T) { + const ( + repo = "repo-for-continuous-export-test" + branch = "main" + anotherBranch = "notMain" + ) + handler, deps := getHandler(t, "") + + creds := createDefaultAdminUser(deps.auth, t) + bauth := httptransport.BasicAuth(creds.AccessKeyID, creds.AccessSecretKey) + + clt := client.Default + clt.SetTransport(&handlerTransport{Handler: handler}) + + ctx := context.Background() + _, err := deps.cataloger.CreateRepository(ctx, repo, "s3://foo1", branch) + testutil.MustDo(t, "create repository", err) + + config := models.ContinuousExportConfiguration{ + ExportPath: strfmt.URI("s3://bucket/export"), + ExportStatusPath: strfmt.URI("s3://bucket/report"), + LastKeysInPrefixRegexp: []string{"^_success$", ".*/_success$"}, + } + + res, err := clt.Export.SetContinuousExport(&export.SetContinuousExportParams{ + Repository: repo, + Branch: branch, + Config: &config, + }, bauth) + testutil.MustDo(t, "initial continuous export configuration", err) + if res == nil { + t.Fatalf("initial continuous export configuration: expected OK but got nil") + } + + t.Run("get missing branch configuration", func(t *testing.T) { + res, err := clt.Export.GetContinuousExport(&export.GetContinuousExportParams{ + Repository: repo, + Branch: anotherBranch, + }, bauth) + if err == nil || res != nil { + t.Fatalf("expected get to return an error but got result %v, error nil", res) + } + if _, ok := err.(*export.GetContinuousExportNotFound); !ok { + t.Errorf("expected get to return not found but got %T %+v", err, err) + } + }) + + t.Run("get configured branch", func(t *testing.T) { + got, err := clt.Export.GetContinuousExport(&export.GetContinuousExportParams{ + Repository: repo, + Branch: branch, + }, bauth) + if err != nil { + t.Fatalf("expected get to return result but got %s", err) + } + if diffs := deep.Equal(config, *got.GetPayload()); diffs != nil { + t.Errorf("got different configuration: %s", diffs) + } + }) + + t.Run("overwrite configuration", func(t *testing.T) { + newConfig := models.ContinuousExportConfiguration{ + ExportPath: strfmt.URI("s3://better-bucket/export"), + ExportStatusPath: strfmt.URI("s3://better-bucket/report"), + LastKeysInPrefixRegexp: nil, + } + _, err := clt.Export.SetContinuousExport(&export.SetContinuousExportParams{ + Repository: repo, + Branch: branch, + Config: &newConfig, + }, bauth) + if err != nil { + t.Errorf("failed to overwrite continuous export configuration: %s", err) + } + got, err := clt.Export.GetContinuousExport(&export.GetContinuousExportParams{ + Repository: repo, + Branch: branch, + }, bauth) + if err != nil { + t.Fatalf("expected get to return result but got %s", err) + } + if diffs := deep.Equal(newConfig, *got.GetPayload()); diffs != nil { + t.Errorf("got different configuration: %s", diffs) + } + }) +} + func Test_setupLakeFSHandler(t *testing.T) { // get handler with DB without apply the DDL handler, deps := getHandler(t, "", testutil.WithGetDBApplyDDL(false)) diff --git a/docs/assets/js/swagger.yml b/docs/assets/js/swagger.yml index 7dd6658e07b..ab2d3ad03dd 100644 --- a/docs/assets/js/swagger.yml +++ b/docs/assets/js/swagger.yml @@ -308,6 +308,35 @@ definitions: - id - statement + continuous_export_configuration: + type: object + required: + - exportPath + properties: + exportPath: + type: string + format: uri + x-nullable: false # Override https://github.com/go-swagger/go-swagger/issues/1188 + # go-swagger totally not a bug. This causes the generated field + # *not* to be a pointer. Then the regular (incorrect, in this + # case) JSON parser parses it as an empty field, and validation + # verifies the value is non-empty. In *this particular case* it + # works because a URI cannot be empty (at least not an absolute + # URI, which is what we require). + description: export objects to this path + example: s3://company-bucket/path/to/export + exportStatusPath: + type: string + format: uri + description: write export status object to this path + example: s3://company-bucket/path/to/status + lastKeysInPrefixRegexp: + type: array + items: + type: string + description: "list of regexps of keys to exported last in each prefix (for signalling)" + example: ["^SUCCESS$", ".*/_SUCCESS$"] + retention_policy: type: object required: @@ -1258,54 +1287,6 @@ paths: schema: $ref: "#/definitions/error" - /repositories/{repository}/inventory/s3/import: - parameters: - - in: path - name: repository - required: true - type: string - - in: query - name: manifestUrl - required: true - type: string - - in: query - name: dryRun - type: boolean - default: false - post: - tags: - - repositories - operationId: importFromS3Inventory - summary: import metadata for an existing bucket in S3 - responses: - 201: - description: import results - schema: - type: object - properties: - is_dry_run: - type: boolean - added_or_changed: - type: integer - format: int64 - deleted: - type: integer - format: int64 - previous_manifest: - type: string - previous_import_date: - type: integer - format: int64 - 401: - $ref: "#/responses/Unauthorized" - 404: - description: "repository not found" - schema: - $ref: "#/definitions/error" - default: - description: generic error response - schema: - $ref: "#/definitions/error" /repositories/{repository}/branches: parameters: - in: path @@ -1972,6 +1953,62 @@ paths: schema: $ref: "#/definitions/error" + /repositories/{repository}/branches/{branch}/continuous-export: + parameters: + - in: path + name: repository + required: true + type: string + - in: path + name: branch + required: true + type: string + get: + tags: + - export + - branches + operationId: getContinuousExport + summary: returns the current continuous export configuration of a branch + responses: + 200: + description: continuous export policy + schema: + $ref: "#/definitions/continuous_export_configuration" + 401: + $ref: "#/responses/Unauthorized" + 404: + description: no continuous export policy defined + schema: + $ref: "#/definitions/error" + default: + description: generic error response + schema: + $ref: "#/definitions/error" + put: + tags: + - export + - branches + operationId: setContinuousExport + summary: sets a new continuous export configuration of a branch + parameters: + - in: body + name: config + required: true + schema: + $ref: "#/definitions/continuous_export_configuration" + responses: + 201: + description: continuous export successfullyconfigured + 401: + $ref: "#/responses/Unauthorized" + 404: + description: no branch defined at that repo + schema: + $ref: "#/definitions/error" + default: + description: generic error response + schema: + $ref: "#/definitions/error" /repositories/{repository}/retention: parameters: diff --git a/swagger.yml b/swagger.yml index 2966ebe6b61..ab2d3ad03dd 100644 --- a/swagger.yml +++ b/swagger.yml @@ -308,6 +308,35 @@ definitions: - id - statement + continuous_export_configuration: + type: object + required: + - exportPath + properties: + exportPath: + type: string + format: uri + x-nullable: false # Override https://github.com/go-swagger/go-swagger/issues/1188 + # go-swagger totally not a bug. This causes the generated field + # *not* to be a pointer. Then the regular (incorrect, in this + # case) JSON parser parses it as an empty field, and validation + # verifies the value is non-empty. In *this particular case* it + # works because a URI cannot be empty (at least not an absolute + # URI, which is what we require). + description: export objects to this path + example: s3://company-bucket/path/to/export + exportStatusPath: + type: string + format: uri + description: write export status object to this path + example: s3://company-bucket/path/to/status + lastKeysInPrefixRegexp: + type: array + items: + type: string + description: "list of regexps of keys to exported last in each prefix (for signalling)" + example: ["^SUCCESS$", ".*/_SUCCESS$"] + retention_policy: type: object required: @@ -1924,6 +1953,62 @@ paths: schema: $ref: "#/definitions/error" + /repositories/{repository}/branches/{branch}/continuous-export: + parameters: + - in: path + name: repository + required: true + type: string + - in: path + name: branch + required: true + type: string + get: + tags: + - export + - branches + operationId: getContinuousExport + summary: returns the current continuous export configuration of a branch + responses: + 200: + description: continuous export policy + schema: + $ref: "#/definitions/continuous_export_configuration" + 401: + $ref: "#/responses/Unauthorized" + 404: + description: no continuous export policy defined + schema: + $ref: "#/definitions/error" + default: + description: generic error response + schema: + $ref: "#/definitions/error" + put: + tags: + - export + - branches + operationId: setContinuousExport + summary: sets a new continuous export configuration of a branch + parameters: + - in: body + name: config + required: true + schema: + $ref: "#/definitions/continuous_export_configuration" + responses: + 201: + description: continuous export successfullyconfigured + 401: + $ref: "#/responses/Unauthorized" + 404: + description: no branch defined at that repo + schema: + $ref: "#/definitions/error" + default: + description: generic error response + schema: + $ref: "#/definitions/error" /repositories/{repository}/retention: parameters: From 56b0acf9ffba4b6b9971c3d0ec10cd562df79c52 Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Mon, 19 Oct 2020 17:06:09 +0300 Subject: [PATCH 3/6] [checks] Format in accordance with golangci - Remove a blank line - Override interfacer change that removes semantic information in exchange for a poorly-named irrelevant interface. --- api/api_controller.go | 1 - catalog/cataloger_export.go | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/api_controller.go b/api/api_controller.go index 8e8a022e8ff..f2a75feead1 100644 --- a/api/api_controller.go +++ b/api/api_controller.go @@ -2230,7 +2230,6 @@ func (c *Controller) ExportGetContinuousExportHandler() exportop.GetContinuousEx LastKeysInPrefixRegexp: prefixRegexps, } return exportop.NewGetContinuousExportOK().WithPayload(&payload) - }) } diff --git a/catalog/cataloger_export.go b/catalog/cataloger_export.go index 0e01cb1c865..c5859e7941c 100644 --- a/catalog/cataloger_export.go +++ b/catalog/cataloger_export.go @@ -118,7 +118,10 @@ func unbracket(s string) (string, error) { // DeconstructDisjunction returns the text forms of the regexps in the disjunction rex. rex // should be constructed (only) by DisjunctRegexps. -func DeconstructDisjunction(regexp *regexp.Regexp) ([]string, error) { +func DeconstructDisjunction(regexp *regexp.Regexp) ([]string, error) { // nolint:interfacer + // Why nollnt above? I really do want this regexp handling function to take a regexp, not + // expvar.Var hich is a silly alias to Stringer. + s := regexp.String() if len(s) == 0 { return nil, nil From 39e6c1ae1265f9b4bdf4a8fd4feb98c6e793398e Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Mon, 19 Oct 2020 18:31:51 +0300 Subject: [PATCH 4/6] Use regexp array directly in LastKeysInPrefixRegexp It is not possible to roundtrip generating and unparsing disjunctions from the database: the regexp parser optimizes and rewrites portions of the regexp. So we need a copy of the input array stored in the database. (Generating a regexp disjunction back from this array will be easy, a small part of the deleted code did just that.) --- api/api_controller.go | 19 +----- catalog/cataloger_export.go | 74 ++++------------------ catalog/cataloger_export_test.go | 105 +++++++++++-------------------- ddl/000008_export_cfg.up.sql | 2 +- go.mod | 2 +- 5 files changed, 53 insertions(+), 149 deletions(-) diff --git a/api/api_controller.go b/api/api_controller.go index f2a75feead1..62fb7f72efb 100644 --- a/api/api_controller.go +++ b/api/api_controller.go @@ -7,7 +7,6 @@ import ( "fmt" "net/http" "path/filepath" - "regexp" "strings" "time" @@ -2211,23 +2210,10 @@ func (c *Controller) ExportGetContinuousExportHandler() exportop.GetContinuousEx WithPayload(responseErrorFrom(err)) } - prefixRegexp, err := regexp.Compile(config.LastKeysInPrefixRegexp) - if err != nil { - return exportop.NewGetContinuousExportDefault(http.StatusInternalServerError). - WithPayload(responseErrorFrom(err)) - } - - prefixRegexps, err := catalog.DeconstructDisjunction(prefixRegexp) - if err != nil { - return exportop.NewGetContinuousExportDefault(http.StatusInternalServerError). - WithPayload(responseErrorFrom(fmt.Errorf( - "cannot deconstruct last_keys_in_prefix_regexp %s: %w", config.LastKeysInPrefixRegexp, err))) - } - payload := models.ContinuousExportConfiguration{ ExportPath: strfmt.URI(config.Path), ExportStatusPath: strfmt.URI(config.StatusPath), - LastKeysInPrefixRegexp: prefixRegexps, + LastKeysInPrefixRegexp: config.LastKeysInPrefixRegexp, } return exportop.NewGetContinuousExportOK().WithPayload(&payload) }) @@ -2248,7 +2234,6 @@ func (c *Controller) ExportSetContinuousExportHandler() exportop.SetContinuousEx deps.LogAction("set_continuous_export") - lastKeysInPrefixRegexp, err := catalog.DisjunctRegexps(params.Config.LastKeysInPrefixRegexp) if err != nil { return exportop.NewSetContinuousExportDefault(http.StatusInternalServerError). WithPayload(responseError("join last keys in prefix regexps: %s", err)) @@ -2256,7 +2241,7 @@ func (c *Controller) ExportSetContinuousExportHandler() exportop.SetContinuousEx config := catalog.ExportConfiguration{ Path: params.Config.ExportPath.String(), StatusPath: params.Config.ExportStatusPath.String(), - LastKeysInPrefixRegexp: lastKeysInPrefixRegexp.String(), + LastKeysInPrefixRegexp: params.Config.LastKeysInPrefixRegexp, } err = deps.Cataloger.PutExportConfiguration(params.Repository, params.Branch, &config) if errors.Is(err, catalog.ErrRepositoryNotFound) || errors.Is(err, catalog.ErrBranchNotFound) { diff --git a/catalog/cataloger_export.go b/catalog/cataloger_export.go index c5859e7941c..23a85262e7c 100644 --- a/catalog/cataloger_export.go +++ b/catalog/cataloger_export.go @@ -1,28 +1,26 @@ package catalog import ( - "errors" "fmt" "regexp" - "regexp/syntax" - "strings" + "github.com/lib/pq" "github.com/treeverse/lakefs/db" ) // ExportConfiguration describes the export configuration of a branch, as passed on wire, used // internally, and stored in DB. type ExportConfiguration struct { - Path string `db:"export_path" json:"exportPath"` - StatusPath string `db:"export_status_path" json:"exportStatusPath"` - LastKeysInPrefixRegexp string `db:"last_keys_in_prefix_regexp" json:"lastKeysInPrefixRegexp"` + Path string `db:"export_path"` + StatusPath string `db:"export_status_path"` + LastKeysInPrefixRegexp pq.StringArray `db:"last_keys_in_prefix_regexp"` } // ExportConfigurationForBranch describes how to export BranchID. It is stored in the database. type ExportConfigurationForBranch struct { ExportConfiguration - Repository string `db:"repository"` - Branch string `db:"branch"` + Repository string + Branch string } func (c *cataloger) GetExportConfigurationForBranch(repository string, branch string) (ExportConfiguration, error) { @@ -66,6 +64,12 @@ func (c *cataloger) GetExportConfigurations() ([]ExportConfigurationForBranch, e } func (c *cataloger) PutExportConfiguration(repository string, branch string, conf *ExportConfiguration) error { + // Validate all fields could be compiled as regexps. + for i, r := range conf.LastKeysInPrefixRegexp { + if _, err := regexp.Compile(r); err != nil { + return fmt.Errorf("invalid regexp /%s/ at position %d in LastKeysInPrefixRegexp: %w", r, i, err) + } + } _, err := c.db.Transact(func(tx db.Tx) (interface{}, error) { branchID, err := c.getBranchIDCache(tx, repository, branch) if err != nil { @@ -83,57 +87,3 @@ func (c *cataloger) PutExportConfiguration(repository string, branch string, con }) return err } - -const ( - groupStart = "(?:" - groupEnd = ")" -) - -// DisjunctRegexps returns a single regexp holding the disjunction ("|") of regexps. -func DisjunctRegexps(regexps []string) (*regexp.Regexp, error) { - parts := make([]string, len(regexps)) - for i, re := range regexps { - _, err := syntax.Parse(re, syntax.Perl) - if err != nil { - return nil, fmt.Errorf("%s: %w", re, err) - } - parts[i] = groupStart + re + groupEnd - } - d := strings.Join(parts, "|") - return regexp.Compile(d) -} - -var ( - ErrNotBracketed = errors.New("not a bracketed string") - unbracketRegexp = regexp.MustCompile("^" + regexp.QuoteMeta(groupStart) + "(.*)" + regexp.QuoteMeta(groupEnd) + "$") -) - -func unbracket(s string) (string, error) { - sub := unbracketRegexp.FindStringSubmatch(s) - if len(sub) == 0 { - return "", fmt.Errorf("%s: %w", s, ErrNotBracketed) - } - return sub[1], nil -} - -// DeconstructDisjunction returns the text forms of the regexps in the disjunction rex. rex -// should be constructed (only) by DisjunctRegexps. -func DeconstructDisjunction(regexp *regexp.Regexp) ([]string, error) { // nolint:interfacer - // Why nollnt above? I really do want this regexp handling function to take a regexp, not - // expvar.Var hich is a silly alias to Stringer. - - s := regexp.String() - if len(s) == 0 { - return nil, nil - } - regexpParts := strings.Split(s, "|") - ret := make([]string, len(regexpParts)) - for i, regexpPart := range regexpParts { - part, err := unbracket(regexpPart) - if err != nil { - return nil, err - } - ret[i] = part - } - return ret, nil -} diff --git a/catalog/cataloger_export_test.go b/catalog/cataloger_export_test.go index a278b6174f1..96057384bbe 100644 --- a/catalog/cataloger_export_test.go +++ b/catalog/cataloger_export_test.go @@ -3,10 +3,11 @@ package catalog import ( "context" "errors" - syntax "regexp/syntax" + "regexp/syntax" "testing" "github.com/go-test/deep" + "github.com/lib/pq" ) const ( @@ -15,72 +16,6 @@ const ( anotherBranch = "lost-not-found" ) -func TestDisjunctRegexps(t *testing.T) { - cases := []struct { - name string - input []string - errCode syntax.ErrorCode - output string - }{ - {name: "empty", output: ""}, - {name: "one", input: []string{"^foo$"}, output: "(?:^foo$)"}, - {name: "two", input: []string{"^foo", "bar$"}, output: "(?:^foo)|(?:bar$)"}, - {name: "many", input: []string{"a", "b", "c", "z"}, output: "(?:a)|(?:b)|(?:c)|(?:z)"}, - {name: "error on first", input: []string{"("}, errCode: syntax.ErrMissingParen}, - {name: "error on second", input: []string{"a", "[z", "!"}, errCode: syntax.ErrMissingBracket}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - got, err := DisjunctRegexps(c.input) - if err != nil { - var regexpErr *syntax.Error - ok := errors.As(err, ®expErr) - if !ok { - t.Errorf("unexpected non-regexp error type %T: %s", err, err) - } else if regexpErr.Code != c.errCode { - t.Errorf("expected regexp code \"%s\" but got \"%s\"", - c.errCode, regexpErr.Code) - } - return - } - if c.errCode != "" { - t.Errorf("expected error \"%s\" but succeeded", c.errCode) - } - if got.String() != c.output { - t.Errorf("expected %s, got %s", c.output, got) - } - }) - } -} - -func TestDeconstructDisjunctionRoundtrip(t *testing.T) { - cases := []struct { - name string - input []string - }{ - {name: "empty"}, - {name: "one", input: []string{"^foo$"}}, - {name: "two", input: []string{"^foo", "bar$"}}, - {name: "many", input: []string{"a", "b", "c", "z"}}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - regexp, err := DisjunctRegexps(c.input) - if err != nil { - t.Fatalf("%s: %s", c.input, err) - } - got, err := DeconstructDisjunction(regexp) - if err != nil { - t.Errorf("deconstruct %s for %v failed: %s", regexp, c.input, err) - } - if diffs := deep.Equal(c.input, got); diffs != nil { - t.Errorf("round-trip %v -> %s: %s", c.input, regexp, diffs) - } - }) - } - -} - func TestExport_GetConfiguration(t *testing.T) { const ( branchID = 17 @@ -93,7 +28,7 @@ func TestExport_GetConfiguration(t *testing.T) { cfg := ExportConfiguration{ Path: "/path/to/export", StatusPath: "/path/to/status", - LastKeysInPrefixRegexp: "*&@!#$", + LastKeysInPrefixRegexp: pq.StringArray{"xyz+y"}, } if err := c.PutExportConfiguration(repo, defaultBranch, &cfg); err != nil { @@ -116,4 +51,38 @@ func TestExport_GetConfiguration(t *testing.T) { t.Errorf("got other configuration than expected: %s", diffs) } }) + + t.Run("reconfigured branch", func(t *testing.T) { + newCfg := ExportConfiguration{ + Path: "/better/to/export", + StatusPath: "/better/for/status", + LastKeysInPrefixRegexp: pq.StringArray{"abc", "def", "xyz"}, + } + if err := c.PutExportConfiguration(repo, defaultBranch, &newCfg); err != nil { + t.Fatalf("update configuration with %+v: %s", newCfg, err) + } + gotCfg, err := c.GetExportConfigurationForBranch(repo, defaultBranch) + if err != nil { + t.Errorf("get updated configuration for configured branch failed: %s", err) + } + if diffs := deep.Equal(newCfg, gotCfg); diffs != nil { + t.Errorf("got other configuration than expected: %s", diffs) + } + }) + + t.Run("invalid regexp", func(t *testing.T) { + badCfg := ExportConfiguration{ + Path: "/better/to/export", + StatusPath: "/better/for/status", + LastKeysInPrefixRegexp: pq.StringArray{"(unclosed"}, + } + err := c.PutExportConfiguration(repo, defaultBranch, &badCfg) + var regexpErr *syntax.Error + if !errors.As(err, ®expErr) { + t.Fatalf("update configuration with bad %+v did not give a regexp error: %s", badCfg, err) + } + if regexpErr.Code != syntax.ErrMissingParen { + t.Errorf("expected configuration update with bad %+v to give missing paren, but got %s", badCfg, regexpErr.Code) + } + }) } diff --git a/ddl/000008_export_cfg.up.sql b/ddl/000008_export_cfg.up.sql index b26e2024817..276126c75cd 100644 --- a/ddl/000008_export_cfg.up.sql +++ b/ddl/000008_export_cfg.up.sql @@ -4,7 +4,7 @@ CREATE TABLE IF NOT EXISTS catalog_branches_export ( branch_id integer PRIMARY KEY, export_path varchar NOT NULL, export_status_path varchar NOT NULL, - last_keys_in_prefix_regexp varchar NOT NULL + last_keys_in_prefix_regexp varchar array ); ALTER TABLE catalog_branches_export diff --git a/go.mod b/go.mod index 306b709c8c7..39f464e88e5 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc github.com/klauspost/compress v1.10.10 // indirect - github.com/lib/pq v1.8.0 // indirect + github.com/lib/pq v1.8.0 github.com/lunixbochs/vtclean v1.0.0 // indirect github.com/mailru/easyjson v0.7.2 // indirect github.com/manifoldco/promptui v0.7.0 From b8ca672ad5e95fe6febc822efd38b60325a5ee02 Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Mon, 19 Oct 2020 19:42:40 +0300 Subject: [PATCH 5/6] Test (and fix...) GetExportConfigurations Not part of the server API so untested. But may be used internally so needs testing. --- catalog/cataloger_export.go | 16 +++++--- catalog/cataloger_export_test.go | 64 +++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/catalog/cataloger_export.go b/catalog/cataloger_export.go index 23a85262e7c..e8a6fa3563d 100644 --- a/catalog/cataloger_export.go +++ b/catalog/cataloger_export.go @@ -17,10 +17,15 @@ type ExportConfiguration struct { } // ExportConfigurationForBranch describes how to export BranchID. It is stored in the database. +// Unfortunately golang sql doesn't know about embedded structs, so you get a useless copy of +// ExportConfiguration embedded here. type ExportConfigurationForBranch struct { - ExportConfiguration - Repository string - Branch string + Repository string `db:"repository"` + Branch string `db:"branch"` + + Path string `db:"export_path"` + StatusPath string `db:"export_status_path"` + LastKeysInPrefixRegexp pq.StringArray `db:"last_keys_in_prefix_regexp"` } func (c *cataloger) GetExportConfigurationForBranch(repository string, branch string) (ExportConfiguration, error) { @@ -48,16 +53,17 @@ func (c *cataloger) GetExportConfigurations() ([]ExportConfigurationForBranch, e `SELECT r.name repository, b.name branch, e.export_path export_path, e.export_status_path export_status_path, e.last_keys_in_prefix_regexp last_keys_in_prefix_regexp - FROM catalog_branches_export e JOIN catalog_branches b ON e.branch_id = b.branch_id + FROM catalog_branches_export e JOIN catalog_branches b ON e.branch_id = b.id JOIN catalog_repositories r ON b.repository_id = r.id`) if err != nil { return nil, err } for rows.Next() { var rec ExportConfigurationForBranch - if err = rows.Scan(&rec); err != nil { + if err = rows.StructScan(&rec); err != nil { return nil, fmt.Errorf("scan configuration %+v: %w", rows, err) } + fmt.Printf("[DEBUG] row %+v\n", rec) ret = append(ret, rec) } return ret, nil diff --git a/catalog/cataloger_export_test.go b/catalog/cataloger_export_test.go index 96057384bbe..dca1e303941 100644 --- a/catalog/cataloger_export_test.go +++ b/catalog/cataloger_export_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "regexp/syntax" + "sort" "testing" "github.com/go-test/deep" @@ -16,7 +17,26 @@ const ( anotherBranch = "lost-not-found" ) -func TestExport_GetConfiguration(t *testing.T) { +// configForBranchSlice adapts a slice to satisfy sort.Interface +type configForBranchSlice []ExportConfigurationForBranch + +func (s configForBranchSlice) Len() int { return len(s) } + +func (s configForBranchSlice) Less(i int, j int) bool { + if s[i].Repository < s[j].Repository { + return true + } + if s[i].Repository > s[j].Repository { + return false + } + return s[i].Branch < s[j].Branch +} + +func (s configForBranchSlice) Swap(i int, j int) { + s[i], s[j] = s[j], s[i] +} + +func TestExportConfiguration(t *testing.T) { const ( branchID = 17 anotherBranchID = 29 @@ -85,4 +105,46 @@ func TestExport_GetConfiguration(t *testing.T) { t.Errorf("expected configuration update with bad %+v to give missing paren, but got %s", badCfg, regexpErr.Code) } }) + + t.Run("GetExportConfigurations", func(t *testing.T) { + moreBranch := "secondary" + if _, err := c.CreateBranch(ctx, repo, moreBranch, defaultBranch); err != nil { + t.Fatalf("create secondary branch: %s", err) + } + moreCfg := ExportConfiguration{ + Path: "/more/to/export", + StatusPath: "/more/for/status", + } + expected := []ExportConfigurationForBranch{ + { + Repository: repo, + Branch: defaultBranch, + Path: cfg.Path, + StatusPath: cfg.StatusPath, + LastKeysInPrefixRegexp: cfg.LastKeysInPrefixRegexp, + }, { + Repository: repo, + Branch: moreBranch, + Path: moreCfg.Path, + StatusPath: moreCfg.StatusPath, + LastKeysInPrefixRegexp: moreCfg.LastKeysInPrefixRegexp, + }, + } + + if err := c.PutExportConfiguration(repo, defaultBranch, &cfg); err != nil { + t.Fatalf("add configuration with %+v failed: %s", cfg, err) + } + if err := c.PutExportConfiguration(repo, moreBranch, &moreCfg); err != nil { + t.Fatalf("add configuration with %+v failed: %s", moreCfg, err) + } + got, err := c.GetExportConfigurations() + if err != nil { + t.Fatal(err) + } + sort.Sort(configForBranchSlice(expected)) + sort.Sort(configForBranchSlice(got)) + if diffs := deep.Equal(expected, got); diffs != nil { + t.Errorf("did not read expected configurations: %s", diffs) + } + }) } From 4245f1126839a9f7fc4e9815ea95eb4b62a8fdaf Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Wed, 21 Oct 2020 17:15:25 +0300 Subject: [PATCH 6/6] [CR] cleanups --- api/api_controller.go | 4 ---- catalog/cataloger_export.go | 1 - ddl/000008_export_cfg.down.sql | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/api/api_controller.go b/api/api_controller.go index 62fb7f72efb..3065e41a8ae 100644 --- a/api/api_controller.go +++ b/api/api_controller.go @@ -2234,10 +2234,6 @@ func (c *Controller) ExportSetContinuousExportHandler() exportop.SetContinuousEx deps.LogAction("set_continuous_export") - if err != nil { - return exportop.NewSetContinuousExportDefault(http.StatusInternalServerError). - WithPayload(responseError("join last keys in prefix regexps: %s", err)) - } config := catalog.ExportConfiguration{ Path: params.Config.ExportPath.String(), StatusPath: params.Config.ExportStatusPath.String(), diff --git a/catalog/cataloger_export.go b/catalog/cataloger_export.go index e8a6fa3563d..f978a60a224 100644 --- a/catalog/cataloger_export.go +++ b/catalog/cataloger_export.go @@ -63,7 +63,6 @@ func (c *cataloger) GetExportConfigurations() ([]ExportConfigurationForBranch, e if err = rows.StructScan(&rec); err != nil { return nil, fmt.Errorf("scan configuration %+v: %w", rows, err) } - fmt.Printf("[DEBUG] row %+v\n", rec) ret = append(ret, rec) } return ret, nil diff --git a/ddl/000008_export_cfg.down.sql b/ddl/000008_export_cfg.down.sql index fa45e5c74c7..8ba5a33a8aa 100644 --- a/ddl/000008_export_cfg.down.sql +++ b/ddl/000008_export_cfg.down.sql @@ -1 +1 @@ -DROP TABLE catalog_branches_export; +DROP TABLE IF EXISTS catalog_branches_export;