diff --git a/api/api_controller.go b/api/api_controller.go index 7e8ca19b39f..9d84e4dd8d8 100644 --- a/api/api_controller.go +++ b/api/api_controller.go @@ -197,7 +197,7 @@ func (c *Controller) Configure(api *operations.LakefsAPI) { api.ExportGetContinuousExportHandler = c.ExportGetContinuousExportHandler() api.ExportSetContinuousExportHandler = c.ExportSetContinuousExportHandler() - api.ExportExecuteContinuousExportHandler = c.ExportExecuteContinuousExport() + api.ExportRunHandler = c.ExportRunHandler() api.ConfigGetConfigHandler = c.ConfigGetConfigHandler() } @@ -2228,8 +2228,8 @@ func (c *Controller) ExportGetContinuousExportHandler() exportop.GetContinuousEx }) } -func (c *Controller) ExportExecuteContinuousExport() exportop.ExecuteContinuousExportHandler { - return exportop.ExecuteContinuousExportHandlerFunc(func(params exportop.ExecuteContinuousExportParams, user *models.User) middleware.Responder { +func (c *Controller) ExportRunHandler() exportop.RunHandler { + return exportop.RunHandlerFunc(func(params exportop.RunParams, user *models.User) middleware.Responder { deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ { Action: permissions.WriteObjectAction, @@ -2242,12 +2242,12 @@ func (c *Controller) ExportExecuteContinuousExport() exportop.ExecuteContinuousE } deps.LogAction("execute_continuous_export") - err = export.ExportBranchStart(deps.Parade, deps.Cataloger, params.Repository, params.Branch) + exportID, err := export.ExportBranchStart(deps.Parade, deps.Cataloger, params.Repository, params.Branch) if err != nil { - return exportop.NewExecuteContinuousExportDefault(http.StatusInternalServerError). + return exportop.NewRunDefault(http.StatusInternalServerError). WithPayload(responseErrorFrom(err)) } - return exportop.NewExecuteContinuousExportCreated() + return exportop.NewRunCreated().WithPayload(exportID) }) } diff --git a/api/client.go b/api/client.go index 1d7ef30d9fe..f56e30e6de7 100644 --- a/api/client.go +++ b/api/client.go @@ -88,7 +88,7 @@ type RepositoryClient interface { SetContinuousExport(ctx context.Context, repository, branchID string, config *models.ContinuousExportConfiguration) error GetContinuousExport(ctx context.Context, repository, branchID string) (*models.ContinuousExportConfiguration, error) - ExecuteContinuousExport(ctx context.Context, repository, branchID string) error + RunExport(ctx context.Context, repository, branchID string) (string, error) } type Client interface { @@ -514,14 +514,17 @@ func (c *client) GetContinuousExport(ctx context.Context, repository, branchID s return resp.GetPayload(), err } -func (c *client) ExecuteContinuousExport(ctx context.Context, repository, branchID string) error { - _, err := c.remote.Export.ExecuteContinuousExport(&export.ExecuteContinuousExportParams{ +func (c *client) RunExport(ctx context.Context, repository, branchID string) (string, error) { + resp, err := c.remote.Export.Run(&export.RunParams{ Branch: branchID, Repository: repository, Context: ctx, HTTPClient: nil, }, c.auth) - return err + if err != nil { + return "", err + } + return resp.GetPayload(), nil } func (c *client) Commit(ctx context.Context, repository, branchID, message string, metadata map[string]string) (*models.Commit, error) { diff --git a/cmd/lakectl/cmd/export.go b/cmd/lakectl/cmd/export.go index c136792024f..bb27aef0332 100644 --- a/cmd/lakectl/cmd/export.go +++ b/cmd/lakectl/cmd/export.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "fmt" "github.com/go-openapi/strfmt" "github.com/treeverse/lakefs/api/gen/models" @@ -81,10 +82,11 @@ var exportExecuteCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { client := getClient() branchURI := uri.Must(uri.Parse(args[0])) - err := client.ExecuteContinuousExport(context.Background(), branchURI.Repository, branchURI.Ref) + exportID, err := client.RunExport(context.Background(), branchURI.Repository, branchURI.Ref) if err != nil { DieErr(err) } + fmt.Printf("Export-ID:%s\n", exportID) }, } diff --git a/docs/assets/js/swagger.yml b/docs/assets/js/swagger.yml index 1d80b89f0af..a45c1e06573 100644 --- a/docs/assets/js/swagger.yml +++ b/docs/assets/js/swagger.yml @@ -2025,11 +2025,14 @@ paths: tags: - export - branches - operationId: startExport + operationId: run summary: hook to be called in order to execute continues export on branch responses: 201: description: continuous export successfully started + schema: + description: "export ID" + type: string 401: $ref: "#/responses/Unauthorized" 404: @@ -2108,3 +2111,4 @@ paths: $ref: "#/definitions/config" 401: $ref: "#/responses/Unauthorized" + diff --git a/export/export.go b/export/export.go index 13883e8e11c..298898c9829 100644 --- a/export/export.go +++ b/export/export.go @@ -21,23 +21,21 @@ func getExportID(repo, branch, commitRef string) (string, error) { // ExportBranchStart inserts a start task on branch, sets branch export state to pending. // If export already in progress will return error -func ExportBranchStart(paradeDB parade.Parade, cataloger catalog.Cataloger, repo, branch string) error { +func ExportBranchStart(paradeDB parade.Parade, cataloger catalog.Cataloger, repo, branch string) (string, error) { commit, err := cataloger.GetCommit(context.Background(), repo, branch) if err != nil { - return err + return "", err } commitRef := commit.Reference - + exportID, err := getExportID(repo, branch, commitRef) + if err != nil { + return "", err + } err = cataloger.ExportState(repo, branch, commitRef, func(oldRef string, state catalog.CatalogBranchExportStatus) (newState catalog.CatalogBranchExportStatus, newMessage *string, err error) { config, err := cataloger.GetExportConfigurationForBranch(repo, branch) if err != nil { return "", nil, err } - - exportID, err := getExportID(repo, branch, commitRef) - if err != nil { - return "", nil, err - } tasks, err := GetStartTasks(repo, branch, oldRef, commitRef, exportID, config) if err != nil { return "", nil, err @@ -49,7 +47,7 @@ func ExportBranchStart(paradeDB parade.Parade, cataloger catalog.Cataloger, repo } return catalog.ExportStatusInProgress, nil, nil }) - return err + return exportID, err } // ExportBranchDone ends the export branch process by changing the status diff --git a/export/export_handler.go b/export/export_handler.go index cef67ef1580..d617fb2041b 100644 --- a/export/export_handler.go +++ b/export/export_handler.go @@ -233,7 +233,7 @@ func (h *Handler) done(body *string, signalledErrors int) error { if err != nil { return err } - return ExportBranchDone(h.cataloger, status, msg, finishData.Branch, finishData.CommitRef, finishData.Repo) + return ExportBranchDone(h.cataloger, status, msg, finishData.Repo, finishData.Branch, finishData.CommitRef) } var errUnknownAction = errors.New("unknown action") diff --git a/export/tasks_generator.go b/export/tasks_generator.go index 1f0d46bd01b..fe02e2d4dc3 100644 --- a/export/tasks_generator.go +++ b/export/tasks_generator.go @@ -121,7 +121,7 @@ func (exportID TaskIDGenerator) makeSuccessTaskID(path string) parade.TaskID { } func (exportID TaskIDGenerator) finishedTaskID() parade.TaskID { - return parade.TaskID(fmt.Sprintf("%s:finished", exportID)) + return parade.TaskID(fmt.Sprintf("%s:finish", exportID)) } func (exportID TaskIDGenerator) startedTaskID() parade.TaskID { diff --git a/export/tasks_generator_test.go b/export/tasks_generator_test.go index 65cb86b617f..357e759d0a3 100644 --- a/export/tasks_generator_test.go +++ b/export/tasks_generator_test.go @@ -230,7 +230,7 @@ func TestTasksGenerator_Empty(t *testing.T) { } doneTasks := getTasks(isDone, tasks) if diffs := deep.Equal(taskPtrs{ - ¶de.TaskData{ID: "empty:finished", Action: export.DoneAction, StatusCode: parade.TaskPending, MaxTries: &one, TotalDependencies: &zero}, + ¶de.TaskData{ID: "empty:finish", Action: export.DoneAction, StatusCode: parade.TaskPending, MaxTries: &one, TotalDependencies: &zero}, }, doneTasks); diffs != nil { t.Error("unexpected done tasks", diffs) } @@ -287,7 +287,7 @@ func TestTasksGenerator_Simple(t *testing.T) { }), StatusCode: parade.TaskPending, TotalDependencies: &zero, - ToSignalAfter: []parade.TaskID{"simple:finished"}, + ToSignalAfter: []parade.TaskID{"simple:finish"}, }, ¶de.TaskData{ ID: idGen.CopyTaskID("change1"), @@ -298,7 +298,7 @@ func TestTasksGenerator_Simple(t *testing.T) { }), StatusCode: parade.TaskPending, TotalDependencies: &zero, - ToSignalAfter: []parade.TaskID{"simple:finished"}, + ToSignalAfter: []parade.TaskID{"simple:finish"}, }, }, copyTasks); diffs != nil { t.Error("unexpected copy tasks", diffs) @@ -313,7 +313,7 @@ func TestTasksGenerator_Simple(t *testing.T) { }), StatusCode: parade.TaskPending, TotalDependencies: &zero, - ToSignalAfter: []parade.TaskID{"simple:finished"}, + ToSignalAfter: []parade.TaskID{"simple:finish"}, }}, deleteTasks); diffs != nil { t.Error("unexpected delete tasks", diffs) } @@ -324,7 +324,7 @@ func TestTasksGenerator_Simple(t *testing.T) { doneTasks := getTasks(isDone, tasks) totalDeps := len(catalogDiffs) if diffs := deep.Equal(taskPtrs{ - ¶de.TaskData{ID: "simple:finished", Action: export.DoneAction, StatusCode: parade.TaskPending, TotalDependencies: &totalDeps}, + ¶de.TaskData{ID: "simple:finish", Action: export.DoneAction, StatusCode: parade.TaskPending, TotalDependencies: &totalDeps}, }, doneTasks); diffs != nil { t.Error("unexpected done tasks", diffs) } @@ -363,25 +363,25 @@ func TestTasksGenerator_SuccessFiles(t *testing.T) { avoid bool }{ {before: idGen.DeleteTaskID("a/success/1"), after: "foo:make-success:a/success"}, - {before: idGen.DeleteTaskID("a/success/1"), after: "foo:finished"}, + {before: idGen.DeleteTaskID("a/success/1"), after: "foo:finish"}, {before: idGen.DeleteTaskID("a/plain/1"), after: "foo:make-success:a/plain", avoid: true}, - {before: idGen.DeleteTaskID("a/plain/1"), after: "foo:finished"}, + {before: idGen.DeleteTaskID("a/plain/1"), after: "foo:finish"}, {before: idGen.DeleteTaskID("a/success/sub/success/11"), after: "foo:make-success:a/success/sub/success"}, - {before: idGen.DeleteTaskID("a/success/sub/success/11"), after: "foo:finished"}, + {before: idGen.DeleteTaskID("a/success/sub/success/11"), after: "foo:finish"}, {before: idGen.DeleteTaskID("a/success/sub/success/12"), after: "foo:make-success:a/success/sub/success"}, - {before: idGen.DeleteTaskID("a/success/sub/success/12"), after: "foo:finished"}, + {before: idGen.DeleteTaskID("a/success/sub/success/12"), after: "foo:finish"}, {before: "foo:make-success:a/success/sub/success", after: "foo:make-success:a/success"}, {before: idGen.DeleteTaskID("b/success/1"), after: "foo:make-success:b/success"}, - {before: idGen.DeleteTaskID("b/success/1"), after: "foo:finished"}, + {before: idGen.DeleteTaskID("b/success/1"), after: "foo:finish"}, {before: idGen.DeleteTaskID("a/success/2"), after: "foo:make-success:a/success"}, - {before: idGen.DeleteTaskID("a/success/2"), after: "foo:finished"}, + {before: idGen.DeleteTaskID("a/success/2"), after: "foo:finish"}, {before: idGen.DeleteTaskID("a/plain/2"), after: "foo:make-success:a/plain", avoid: true}, - {before: idGen.DeleteTaskID("a/plain/2"), after: "foo:finished"}, + {before: idGen.DeleteTaskID("a/plain/2"), after: "foo:finish"}, } gen := export.NewTasksGenerator( "foo", diff --git a/swagger.yml b/swagger.yml index a5919eb2847..a45c1e06573 100644 --- a/swagger.yml +++ b/swagger.yml @@ -2025,11 +2025,14 @@ paths: tags: - export - branches - operationId: executeContinuousExport + operationId: run summary: hook to be called in order to execute continues export on branch responses: 201: description: continuous export successfully started + schema: + description: "export ID" + type: string 401: $ref: "#/responses/Unauthorized" 404: