From 79ff970ec6a526e058a654284b3fba7f2737174e Mon Sep 17 00:00:00 2001 From: guyhardonag Date: Thu, 19 Nov 2020 17:21:35 +0200 Subject: [PATCH] add repair option to export when running export with repair it will consider the previous errors as repaired and continue from last commit --- api/api_controller.go | 8 ++++++-- api/client.go | 5 +++-- catalog/cataloger_export.go | 1 + cmd/lakectl/cmd/export.go | 8 +++++++- ddl/000013_export_repair.down.sql | 3 +++ ddl/000013_export_repair.up.sql | 1 + docs/assets/js/swagger.yml | 4 ++++ export/export.go | 17 +++++++++++++++++ swagger.yml | 4 ++++ 9 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 ddl/000013_export_repair.down.sql create mode 100644 ddl/000013_export_repair.up.sql diff --git a/api/api_controller.go b/api/api_controller.go index 62863e92291..a2c82f218c5 100644 --- a/api/api_controller.go +++ b/api/api_controller.go @@ -2230,8 +2230,12 @@ func (c *Controller) ExportRunHandler() exportop.RunHandler { WithPayload(responseErrorFrom(err)) } deps.LogAction("execute_continuous_export") - - exportID, err := export.ExportBranchStart(deps.Parade, deps.Cataloger, params.Repository, params.Branch) + var exportID string + if swag.BoolValue(params.Repair) { + exportID, err = export.ExportBranchRepair(deps.Parade, deps.Cataloger, params.Repository, params.Branch) + } else { + exportID, err = export.ExportBranchStart(deps.Parade, deps.Cataloger, params.Repository, params.Branch) + } if err != nil { return exportop.NewRunDefault(http.StatusInternalServerError). WithPayload(responseErrorFrom(err)) diff --git a/api/client.go b/api/client.go index f56e30e6de7..0cd2906e716 100644 --- a/api/client.go +++ b/api/client.go @@ -88,7 +88,7 @@ type RepositoryClient interface { SetContinuousExport(ctx context.Context, repository, branchID string, config *models.ContinuousExportConfiguration) error GetContinuousExport(ctx context.Context, repository, branchID string) (*models.ContinuousExportConfiguration, error) - RunExport(ctx context.Context, repository, branchID string) (string, error) + RunExport(ctx context.Context, repository, branchID string, repair *bool) (string, error) } type Client interface { @@ -514,10 +514,11 @@ func (c *client) GetContinuousExport(ctx context.Context, repository, branchID s return resp.GetPayload(), err } -func (c *client) RunExport(ctx context.Context, repository, branchID string) (string, error) { +func (c *client) RunExport(ctx context.Context, repository, branchID string, repair *bool) (string, error) { resp, err := c.remote.Export.Run(&export.RunParams{ Branch: branchID, Repository: repository, + Repair: repair, Context: ctx, HTTPClient: nil, }, c.auth) diff --git a/catalog/cataloger_export.go b/catalog/cataloger_export.go index 91c5c0f5db4..e1fbbaa8958 100644 --- a/catalog/cataloger_export.go +++ b/catalog/cataloger_export.go @@ -40,6 +40,7 @@ const ( ExportStatusInProgress = CatalogBranchExportStatus("in-progress") ExportStatusSuccess = CatalogBranchExportStatus("exported-successfully") ExportStatusFailed = CatalogBranchExportStatus("export-failed") + ExportStatusRepaired = CatalogBranchExportStatus("export-repaired") ExportStatusUnknown = CatalogBranchExportStatus("[unknown]") ) diff --git a/cmd/lakectl/cmd/export.go b/cmd/lakectl/cmd/export.go index 4f9f4b791a8..e3e835ff42e 100644 --- a/cmd/lakectl/cmd/export.go +++ b/cmd/lakectl/cmd/export.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "github.com/go-openapi/swag" "github.com/go-openapi/strfmt" "github.com/treeverse/lakefs/api/gen/models" @@ -96,7 +97,11 @@ var exportExecuteCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { client := getClient() branchURI := uri.Must(uri.Parse(args[0])) - exportID, err := client.RunExport(context.Background(), branchURI.Repository, branchURI.Ref) + repair, err := cmd.Flags().GetBool("repair") + if err != nil { + DieErr(err) + } + exportID, err := client.RunExport(context.Background(), branchURI.Repository, branchURI.Ref, swag.Bool(repair)) if err != nil { DieErr(err) } @@ -117,4 +122,5 @@ func init() { exportSetCmd.Flags().Bool("continuous", false, "export branch after every commit or merge (...=false to disable)") _ = exportSetCmd.MarkFlagRequired("path") _ = exportSetCmd.MarkFlagRequired("continuous") + exportExecuteCmd.Flags().Bool("repair", false, "repair will change current state from failed to repaired before starting a new export") } diff --git a/ddl/000013_export_repair.down.sql b/ddl/000013_export_repair.down.sql new file mode 100644 index 00000000000..d306f7dd76d --- /dev/null +++ b/ddl/000013_export_repair.down.sql @@ -0,0 +1,3 @@ +update catalog_branches_export_state +set state='exported-successfully' +where state = 'export-repaired' \ No newline at end of file diff --git a/ddl/000013_export_repair.up.sql b/ddl/000013_export_repair.up.sql new file mode 100644 index 00000000000..ffdff9629f4 --- /dev/null +++ b/ddl/000013_export_repair.up.sql @@ -0,0 +1 @@ +ALTER type catalog_branch_export_status ADD VALUE IF NOT EXISTS 'export-repaired' \ No newline at end of file diff --git a/docs/assets/js/swagger.yml b/docs/assets/js/swagger.yml index 1ac5d5ce5ae..352aac950e7 100644 --- a/docs/assets/js/swagger.yml +++ b/docs/assets/js/swagger.yml @@ -2024,6 +2024,10 @@ paths: name: branch required: true type: string + - in: query + description: will change export status from export-failed to repaired and start a new export + name: repair + type: boolean post: tags: - export diff --git a/export/export.go b/export/export.go index 8dfaae63bf3..817361a7967 100644 --- a/export/export.go +++ b/export/export.go @@ -66,3 +66,20 @@ func ExportBranchDone(cataloger catalog.Cataloger, status catalog.CatalogBranchE }) return err } + +var ErrRepairNonFailed = errors.New("branch status is not export-failed, could not be repaired") + +// ExportBranchRepair will change state from Failed To Repair and start a new export +// will return Error in Case current state is not Failed +func ExportBranchRepair(paradeDB parade.Parade, cataloger catalog.Cataloger, repo, branch string) (string, error) { + err := cataloger.ExportState(repo, branch, "", func(oldRef string, state catalog.CatalogBranchExportStatus) (newState catalog.CatalogBranchExportStatus, newMessage *string, err error) { + if state != catalog.ExportStatusFailed { + return "", nil, ErrRepairNonFailed + } + return catalog.ExportStatusRepaired, nil, nil + }) + if err != nil { + return "", err + } + return ExportBranchStart(paradeDB, cataloger, repo, branch) +} diff --git a/swagger.yml b/swagger.yml index 1ac5d5ce5ae..d651112a22e 100644 --- a/swagger.yml +++ b/swagger.yml @@ -2024,6 +2024,10 @@ paths: name: branch required: true type: string + - in: query + description: will change export status from export-failed to repaired + name: repair + type: boolean post: tags: - export