diff --git a/dkron/agent.go b/dkron/agent.go index f5ffacf5b..310141648 100644 --- a/dkron/agent.go +++ b/dkron/agent.go @@ -826,3 +826,21 @@ func (a *Agent) GetActiveExecutions() ([]*proto.Execution, error) { return executions, nil } + +func (a *Agent) recursiveSetJob(jobs []*Job) []string { + result := make([]string, 0) + for _, job := range jobs { + err := a.GRPCClient.SetJob(job) + if err != nil { + result = append(result, "fail create "+job.Name) + continue + } else { + result = append(result, "success create "+job.Name) + if len(job.ChildJobs) > 0 { + recursiveResult := a.recursiveSetJob(job.ChildJobs) + result = append(result, recursiveResult...) + } + } + } + return result +} diff --git a/dkron/api.go b/dkron/api.go index 9f467641d..f13d9cd94 100644 --- a/dkron/api.go +++ b/dkron/api.go @@ -1,7 +1,9 @@ package dkron import ( + "encoding/json" "fmt" + "io/ioutil" "net/http" "github.com/gin-contrib/expvar" @@ -69,6 +71,7 @@ func (h *HTTPTransport) APIRoutes(r *gin.RouterGroup, middleware ...gin.HandlerF v1.GET("/leader", h.leaderHandler) v1.GET("/isleader", h.isLeaderHandler) v1.POST("/leave", h.leaveHandler) + v1.POST("/restore", h.restoreHandler) v1.GET("/busy", h.busyHandler) @@ -218,6 +221,42 @@ func (h *HTTPTransport) jobRunHandler(c *gin.Context) { renderJSON(c, http.StatusOK, job) } +// Restore jobs from file. +// Overwrite job if the job is exist. +func (h *HTTPTransport) restoreHandler(c *gin.Context) { + file, _, err := c.Request.FormFile("file") + if err != nil { + c.AbortWithError(http.StatusNotFound, err) + return + } + + data, err := ioutil.ReadAll(file) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + var jobs []*Job + err = json.Unmarshal(data, &jobs) + + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + jobTree, err := generateJobTree(jobs) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + result := h.agent.recursiveSetJob(jobTree) + resp, err := json.Marshal(result) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + renderJSON(c, http.StatusOK, string(resp)) +} + func (h *HTTPTransport) executionsHandler(c *gin.Context) { jobName := c.Param("job") diff --git a/dkron/api_test.go b/dkron/api_test.go index fd04a9a2e..d9aabb06a 100644 --- a/dkron/api_test.go +++ b/dkron/api_test.go @@ -4,9 +4,12 @@ import ( "bytes" "encoding/json" "fmt" + "io" "io/ioutil" + "mime/multipart" "net/http" "os" + "strings" "testing" "time" @@ -275,6 +278,43 @@ func TestAPIJobCreateUpdateJobWithInvalidParentIsNotCreated(t *testing.T) { assert.Equal(t, http.StatusNotFound, resp.StatusCode) } +func TestAPIJobRestore(t *testing.T) { + port := "8109" + baseURL := fmt.Sprintf("http://localhost:%s/v1/restore", port) + dir, a := setupAPITest(t, port) + defer os.RemoveAll(dir) + defer a.Stop() + + bodyBuffer := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuffer) + + fileWriter, err := bodyWriter.CreateFormFile("file", "testBackupJobs.json") + if err != nil { + t.Fatalf("CreateFormFile error: %s", err) + } + + file, err := os.Open("../scripts/testBackupJobs.json") + if err != nil { + t.Fatalf("open job json file error: %s", err) + } + defer file.Close() + + io.Copy(fileWriter, file) + + contentType := bodyWriter.FormDataContentType() + bodyWriter.Close() + + resp, _ := http.Post(baseURL, contentType, bodyBuffer) + respBody, _ := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + rs := string(respBody) + t.Log("restore response: ", rs) + if strings.Contains(rs, "fail") { + t.Fatalf("restore json file request error: %s", rs) + } + +} + // postJob POSTs the given json to the jobs endpoint and returns the response func postJob(t *testing.T, port string, jsonStr []byte) *http.Response { baseURL := fmt.Sprintf("http://localhost:%s/v1", port) diff --git a/dkron/job.go b/dkron/job.go index 486d86ac6..9ce79e54e 100644 --- a/dkron/job.go +++ b/dkron/job.go @@ -102,6 +102,9 @@ type Job struct { // Jobs that are dependent upon this one will be run after this job runs. DependentJobs []string `json:"dependent_jobs"` + // Job pointer that are dependent upon this one + ChildJobs []*Job `json:"-"` + // Job id of job that this job is dependent upon. ParentJob string `json:"parent_job"` @@ -365,3 +368,68 @@ func isSlug(candidate string) (bool, string) { whyNot := illegalCharPattern.FindString(candidate) return whyNot == "", whyNot } + +// generate Job Tree +func generateJobTree(jobs []*Job) ([]*Job, error) { + length := len(jobs) + j := 0 + for i := 0; i < length; i++ { + rejobs, isTopParentNodeFlag, err := findParentJobAndValidateJob(jobs, j) + if err != nil { + return nil, err + } + if isTopParentNodeFlag { + j++ + } + jobs = rejobs + } + return jobs, nil +} + +// findParentJobAndValidateJob... +func findParentJobAndValidateJob(jobs []*Job, index int) ([]*Job, bool, error) { + childJob := jobs[index] + // Validate job + if err := childJob.Validate(); err != nil { + return nil, false, err + } + if childJob.ParentJob == "" { + return jobs, true, nil + } + for _, parentJob := range jobs { + if parentJob.Name == childJob.Name { + continue + } + if childJob.ParentJob == parentJob.Name { + parentJob.ChildJobs = append(parentJob.ChildJobs, childJob) + jobs = append(jobs[:index], jobs[index+1:]...) + return jobs, false, nil + } + if len(parentJob.ChildJobs) > 0 { + flag := findParentJobInChildJobs(parentJob.ChildJobs, childJob) + if flag { + jobs = append(jobs[:index], jobs[index+1:]...) + return jobs, false, nil + } + } + } + return nil, false, ErrNoParent +} + +func findParentJobInChildJobs(jobs []*Job, job *Job) bool { + for _, parentJob := range jobs { + if job.ParentJob == parentJob.Name { + parentJob.ChildJobs = append(parentJob.ChildJobs, job) + return true + } else { + if len(parentJob.ChildJobs) > 0 { + flag := findParentJobInChildJobs(parentJob.ChildJobs, job) + if flag { + return true + } + } + + } + } + return false +} diff --git a/dkron/job_test.go b/dkron/job_test.go index cf286c1d1..fb48bec54 100644 --- a/dkron/job_test.go +++ b/dkron/job_test.go @@ -1,6 +1,7 @@ package dkron import ( + "encoding/json" "testing" "time" @@ -189,3 +190,282 @@ func (gRPCClientMock) GetActiveExecutions(s string) ([]*proto.Execution, error) }, nil } func (gRPCClientMock) SetExecution(execution *proto.Execution) error { return nil } + +func Test_generateJobTree(t *testing.T) { + jsonString := `[ + { + "name": "task1", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-11-28T09:00:00Z" + }, + { + "name": "task2", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-11-28T09:00:00Z" + }, + { + "name": "task3", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "task2", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-11-28T09:00:00Z" + }, + { + "name": "task4", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-11-28T09:00:00Z" + }, + { + "name": "task5", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "task4", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-11-28T09:00:00Z" + }, + { + "name": "task6", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "task5", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-11-28T09:00:00Z" + }, + { + "name": "task7", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "task5", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-11-28T09:00:00Z" + }, + { + "name": "task8", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "task6", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-11-28T09:00:00Z" + } + ]` + var jobs []*Job + err := json.Unmarshal([]byte(jsonString), &jobs) + if err != nil { + t.Fatalf("unmarshal json job error: %s", err) + } + jobTree, err := generateJobTree(jobs) + if err != nil { + t.Fatalf("generate job tree error: %s", err) + } + assert.Equal(t, len(jobTree), 3) +} diff --git a/scripts/testBackupJobs.json b/scripts/testBackupJobs.json new file mode 100644 index 000000000..20b628635 --- /dev/null +++ b/scripts/testBackupJobs.json @@ -0,0 +1,275 @@ +[ + { + "name": "task1", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-12-09T09:00:00Z" + }, + { + "name": "task2", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": [ + "task3" + ], + "parent_job": "", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-12-09T09:00:00Z" + }, + { + "name": "task3", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "task2", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-12-09T09:00:00Z" + }, + { + "name": "task4", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": [ + "task5" + ], + "parent_job": "", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-12-09T09:00:00Z" + }, + { + "name": "task5", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": [ + "task6", + "task7" + ], + "parent_job": "task4", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-12-09T09:00:00Z" + }, + { + "name": "task6", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": [ + "task8" + ], + "parent_job": "task5", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-12-09T09:00:00Z" + }, + { + "name": "task7", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "task5", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-12-09T09:00:00Z" + }, + { + "name": "task8", + "displayname": "", + "timezone": "", + "schedule": "0 0 17 * * *", + "owner": "", + "owner_email": "", + "success_count": 26, + "error_count": 6, + "last_success": "2019-11-04T04:37:12.396866367Z", + "last_error": "2019-11-27T15:17:47.925761Z", + "disabled": false, + "tags": null, + "metadata": null, + "retries": 0, + "dependent_jobs": null, + "parent_job": "task6", + "processors": {}, + "concurrency": "forbid", + "executor": "http", + "executor_config": { + "body": "{\"partner_id\": \"3123\",\"user_id\": \"14123\"}", + "debug": "true", + "expectBody": "", + "expectCode": "200", + "headers": "[]", + "method": "POST", + "timeout": "30", + "url": "xxxxxxxxxxxxx" + }, + "status": "success", + "next": "2019-12-09T09:00:00Z" + } +] \ No newline at end of file diff --git a/website/content/swagger.yaml b/website/content/swagger.yaml index a28d75642..8ec4acd95 100644 --- a/website/content/swagger.yaml +++ b/website/content/swagger.yaml @@ -146,6 +146,26 @@ paths: description: Successful response schema: $ref: '#/definitions/job' + /restore: + post: + description: | + Restore jobs from json file. + operationId: restore + tags: + - jobs + parameters: + - in: formData + name: file + description: Json file that needs to be restored. + required: true + type: file + responses: + 200: + description: Successful response + schema: + type: array + items: + $ref: '#/definitions/restore' /members: get: description: | @@ -459,3 +479,9 @@ definitions: example: files: forward: true + + restore: + type: string + description: Each job restore result. + example: "success create job_1" +