add repair option to export

when running export with repair it will consider the previous errors as repaired and continue from last commit

api/api_controller.go

@@ -2230,8 +2230,12 @@ func (c *Controller) ExportRunHandler() exportop.RunHandler {
 				WithPayload(responseErrorFrom(err))
 		}
 		deps.LogAction("execute_continuous_export")
-
-		exportID, err := export.ExportBranchStart(deps.Parade, deps.Cataloger, params.Repository, params.Branch)
+		var exportID string
+		if swag.BoolValue(params.Repair) {
+			exportID, err = export.ExportBranchRepair(deps.Parade, deps.Cataloger, params.Repository, params.Branch)
+		} else {
+			exportID, err = export.ExportBranchStart(deps.Parade, deps.Cataloger, params.Repository, params.Branch)
+		}
 		if err != nil {
 			return exportop.NewRunDefault(http.StatusInternalServerError).
 				WithPayload(responseErrorFrom(err))

api/client.go

@@ -88,7 +88,7 @@ type RepositoryClient interface {
 
 	SetContinuousExport(ctx context.Context, repository, branchID string, config *models.ContinuousExportConfiguration) error
 	GetContinuousExport(ctx context.Context, repository, branchID string) (*models.ContinuousExportConfiguration, error)
-	RunExport(ctx context.Context, repository, branchID string) (string, error)
+	RunExport(ctx context.Context, repository, branchID string, repair *bool) (string, error)
 }
 
 type Client interface {
@@ -514,10 +514,11 @@ func (c *client) GetContinuousExport(ctx context.Context, repository, branchID s
 	return resp.GetPayload(), err
 }
 
-func (c *client) RunExport(ctx context.Context, repository, branchID string) (string, error) {
+func (c *client) RunExport(ctx context.Context, repository, branchID string, repair *bool) (string, error) {
 	resp, err := c.remote.Export.Run(&export.RunParams{
 		Branch:     branchID,
 		Repository: repository,
+		Repair:     repair,
 		Context:    ctx,
 		HTTPClient: nil,
 	}, c.auth)

catalog/cataloger_export.go

@@ -40,6 +40,7 @@ const (
 	ExportStatusInProgress = CatalogBranchExportStatus("in-progress")
 	ExportStatusSuccess    = CatalogBranchExportStatus("exported-successfully")
 	ExportStatusFailed     = CatalogBranchExportStatus("export-failed")
+	ExportStatusRepaired   = CatalogBranchExportStatus("export-repaired")
 	ExportStatusUnknown    = CatalogBranchExportStatus("[unknown]")
 )
 

cmd/lakectl/cmd/export.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"github.com/go-openapi/swag"
 
 	"github.com/go-openapi/strfmt"
 	"github.com/treeverse/lakefs/api/gen/models"
@@ -96,7 +97,11 @@ var exportExecuteCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		client := getClient()
 		branchURI := uri.Must(uri.Parse(args[0]))
-		exportID, err := client.RunExport(context.Background(), branchURI.Repository, branchURI.Ref)
+		repair, err := cmd.Flags().GetBool("repair")
+		if err != nil {
+			DieErr(err)
+		}
+		exportID, err := client.RunExport(context.Background(), branchURI.Repository, branchURI.Ref, swag.Bool(repair))
 		if err != nil {
 			DieErr(err)
 		}
@@ -117,4 +122,5 @@ func init() {
 	exportSetCmd.Flags().Bool("continuous", false, "export branch after every commit or merge (...=false to disable)")
 	_ = exportSetCmd.MarkFlagRequired("path")
 	_ = exportSetCmd.MarkFlagRequired("continuous")
+	exportExecuteCmd.Flags().Bool("repair", false, "repair will change current state from failed to repaired before starting a new export")
 }

ddl/000013_export_repair.down.sql

@@ -0,0 +1,3 @@
+update catalog_branches_export_state
+set state='exported-successfully'
+where state = 'export-repaired'

ddl/000013_export_repair.up.sql

@@ -0,0 +1 @@
+ALTER type catalog_branch_export_status ADD VALUE IF NOT EXISTS 'export-repaired'

docs/assets/js/swagger.yml

@@ -2024,6 +2024,10 @@ paths:
         name: branch
         required: true
         type: string
+      - in: query
+        description: will change export status from export-failed to repaired and start a new export
+        name: repair
+        type: boolean
     post:
       tags:
         - export

export/export.go

@@ -66,3 +66,20 @@ func ExportBranchDone(cataloger catalog.Cataloger, status catalog.CatalogBranchE
 	})
 	return err
 }
+
+var ErrRepairNonFailed = errors.New("branch status is not export-failed, could not be repaired")
+
+// ExportBranchRepair will change state from Failed To Repair and start a new export
+// will return Error in Case current state is not Failed
+func ExportBranchRepair(paradeDB parade.Parade, cataloger catalog.Cataloger, repo, branch string) (string, error) {
+	err := cataloger.ExportState(repo, branch, "", func(oldRef string, state catalog.CatalogBranchExportStatus) (newState catalog.CatalogBranchExportStatus, newMessage *string, err error) {
+		if state != catalog.ExportStatusFailed {
+			return "", nil, ErrRepairNonFailed
+		}
+		return catalog.ExportStatusRepaired, nil, nil
+	})
+	if err != nil {
+		return "", err
+	}
+	return ExportBranchStart(paradeDB, cataloger, repo, branch)
+}

swagger.yml

@@ -2024,6 +2024,10 @@ paths:
         name: branch
         required: true
         type: string
+      - in: query
+        description: will change export status from export-failed to repaired
+        name: repair
+        type: boolean
     post:
       tags:
         - export