From 717a50f9e32ed45dc0d877e7e7a9919be9cd208c Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Wed, 17 Jul 2024 16:58:36 +0800 Subject: [PATCH 1/3] fix: circleci date fields are inconsistent --- .../20240717_add_createddate_to_workflow.go | 51 +++++++++++++++++++ .../models/migrationscripts/register.go | 1 + backend/plugins/circleci/models/pipeline.go | 16 +++--- backend/plugins/circleci/models/workflow.go | 4 +- .../circleci/tasks/workflow_converter.go | 4 +- .../circleci/tasks/workflow_extractor.go | 7 +-- 6 files changed, 68 insertions(+), 15 deletions(-) create mode 100644 backend/plugins/circleci/models/migrationscripts/20240717_add_createddate_to_workflow.go diff --git a/backend/plugins/circleci/models/migrationscripts/20240717_add_createddate_to_workflow.go b/backend/plugins/circleci/models/migrationscripts/20240717_add_createddate_to_workflow.go new file mode 100644 index 00000000000..b17d3c0ec5e --- /dev/null +++ b/backend/plugins/circleci/models/migrationscripts/20240717_add_createddate_to_workflow.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" +) + +type workflow20240717 struct { + CreatedDate *time.Time +} + +func (workflow20240717) TableName() string { + return "_tool_circleci_workflows" +} + +type addCreatedStoppedDateToWorkflow struct{} + +func (*addCreatedStoppedDateToWorkflow) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.AutoMigrate(&workflow20240717{}); err != nil { + return err + } + return db.RenameColumn(workflow20240717{}.TableName(), "stopped_at", "stopped_date") +} + +func (*addCreatedStoppedDateToWorkflow) Version() uint64 { + return 20231129211000 +} + +func (*addCreatedStoppedDateToWorkflow) Name() string { + return "add created_date and stopped_date to _tool_circleci_workflows, drop stopped_at column" +} diff --git a/backend/plugins/circleci/models/migrationscripts/register.go b/backend/plugins/circleci/models/migrationscripts/register.go index 5a23f27cc85..0233b3b2b06 100644 --- a/backend/plugins/circleci/models/migrationscripts/register.go +++ b/backend/plugins/circleci/models/migrationscripts/register.go @@ -24,5 +24,6 @@ func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables), new(addFieldsToCircleciJob20231129), + new(addCreatedStoppedDateToWorkflow), } } diff --git a/backend/plugins/circleci/models/pipeline.go b/backend/plugins/circleci/models/pipeline.go index a18cded109d..80cd4749cd2 100644 --- a/backend/plugins/circleci/models/pipeline.go +++ b/backend/plugins/circleci/models/pipeline.go @@ -46,14 +46,14 @@ type CircleciVcs struct { } type CircleciPipeline struct { - ConnectionId uint64 `gorm:"primaryKey;type:BIGINT" json:"connectionId" mapstructure:"connectionId"` - Id string `gorm:"primaryKey;type:varchar(100)" json:"id" mapstructure:"id"` - ProjectSlug string `gorm:"type:varchar(255)" json:"projectSlug" mapstructure:"projectSlug"` - UpdatedDate *common.Iso8601Time `json:"updatedDate" mapstructure:"updatedDate"` - CreatedDate *common.Iso8601Time `json:"createdDate" mapstructure:"createdDate"` - Number int64 `json:"number" mapstructure:"number"` // pipeline number within the project? - TriggerParameters any `gorm:"serializer:json" json:"trigger_parameters" mapstructure:"triggerParameters"` - State string `gorm:"type:varchar(100)" json:"state" mapstructure:"state"` + ConnectionId uint64 `gorm:"primaryKey;type:BIGINT" json:"connectionId"` + Id string `gorm:"primaryKey;type:varchar(100)" json:"id"` + ProjectSlug string `gorm:"type:varchar(255)" json:"project_slug"` + UpdatedDate *common.Iso8601Time `json:"updated_at"` + CreatedDate *common.Iso8601Time `json:"created_at"` + Number int64 `json:"number"` // pipeline number within the project? + TriggerParameters any `gorm:"serializer:json" json:"trigger_parameters"` + State string `gorm:"type:varchar(100)" json:"state"` Trigger CircleciTrigger `gorm:"serializer:json" json:"trigger"` Vcs CircleciVcs `gorm:"serializer:json" json:"vcs"` common.NoPKModel `swaggerignore:"true" json:"-" mapstructure:"-"` diff --git a/backend/plugins/circleci/models/workflow.go b/backend/plugins/circleci/models/workflow.go index f4e5e9f7111..c70ee1a77a4 100644 --- a/backend/plugins/circleci/models/workflow.go +++ b/backend/plugins/circleci/models/workflow.go @@ -33,8 +33,8 @@ type CircleciWorkflow struct { Status string `gorm:"type:varchar(100)" json:"status"` StartedBy string `gorm:"type:varchar(100)" json:"started_by"` PipelineNumber int64 `json:"pipeline_number"` - CreatedAt *common.Iso8601Time `json:"created_at"` - StoppedAt *common.Iso8601Time `json:"stopped_at"` + CreatedDate *common.Iso8601Time `json:"created_at"` + StoppedDate *common.Iso8601Time `json:"stopped_at"` DurationSec float64 `json:"duration_sec"` common.NoPKModel `swaggerignore:"true" json:"-" mapstructure:"-"` diff --git a/backend/plugins/circleci/tasks/workflow_converter.go b/backend/plugins/circleci/tasks/workflow_converter.go index 91806fad545..412bd4181c8 100644 --- a/backend/plugins/circleci/tasks/workflow_converter.go +++ b/backend/plugins/circleci/tasks/workflow_converter.go @@ -57,7 +57,7 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error { Input: cursor, Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { userTool := inputRow.(*models.CircleciWorkflow) - createdAt := userTool.CreatedAt.ToTime() + createdAt := userTool.CreatedDate.ToTime() pipeline := &devops.CICDPipeline{ DomainEntity: domainlayer.DomainEntity{ Id: getPipelineIdGen().Generate(data.Options.ConnectionId, userTool.Id), @@ -67,7 +67,7 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error { TaskDatesInfo: devops.TaskDatesInfo{ CreatedDate: createdAt, StartedDate: &createdAt, - FinishedDate: userTool.StoppedAt.ToNullableTime(), + FinishedDate: userTool.StoppedDate.ToNullableTime(), }, CicdScopeId: getProjectIdGen().Generate(data.Options.ConnectionId, userTool.ProjectSlug), // reference: https://circleci.com/docs/api/v2/index.html#operation/getWorkflowById diff --git a/backend/plugins/circleci/tasks/workflow_extractor.go b/backend/plugins/circleci/tasks/workflow_extractor.go index ba0639bc877..37d22f30742 100644 --- a/backend/plugins/circleci/tasks/workflow_extractor.go +++ b/backend/plugins/circleci/tasks/workflow_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" @@ -48,9 +49,9 @@ func ExtractWorkflows(taskCtx plugin.SubTaskContext) errors.Error { toolL := userRes toolL.ConnectionId = data.Options.ConnectionId toolL.ProjectSlug = data.Options.ProjectSlug - if userRes.CreatedAt != nil && userRes.StoppedAt != nil { - startTime := userRes.CreatedAt.ToTime() - endTime := userRes.StoppedAt.ToTime() + if userRes.CreatedDate != nil && userRes.StoppedDate != nil { + startTime := userRes.CreatedDate.ToTime() + endTime := userRes.StoppedDate.ToTime() toolL.DurationSec = float64(endTime.Sub(startTime).Milliseconds() / 1e3) } return []interface{}{ From bbbc577629b811830af05dc62a173e8bbd9b6e97 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Wed, 17 Jul 2024 17:31:53 +0800 Subject: [PATCH 2/3] fix: e2e test --- backend/helpers/e2ehelper/data_flow_tester.go | 7 ++++++ backend/plugins/circleci/e2e/pipeline_test.go | 2 +- .../_tool_circleci_pipelines.csv | 22 +++++++++---------- .../_tool_circleci_workflows.csv | 20 ++++++++--------- 4 files changed, 29 insertions(+), 22 deletions(-) diff --git a/backend/helpers/e2ehelper/data_flow_tester.go b/backend/helpers/e2ehelper/data_flow_tester.go index 831b859b898..b409fd3604a 100644 --- a/backend/helpers/e2ehelper/data_flow_tester.go +++ b/backend/helpers/e2ehelper/data_flow_tester.go @@ -319,6 +319,13 @@ func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, opts TableOptions) { } allFields := append(pkColumnNames, targetFields...) allFields = utils.StringsUniq(allFields) + for i, field := range allFields { + if t.Dal.Dialect() == "mysql" { + allFields[i] = fmt.Sprintf("`%s`", field) + } else { + allFields[i] = fmt.Sprintf(`"%s"`, field) + } + } dbCursor, err := t.Dal.Cursor( dal.Select(strings.Join(allFields, `,`)), dal.From(dst.TableName()), diff --git a/backend/plugins/circleci/e2e/pipeline_test.go b/backend/plugins/circleci/e2e/pipeline_test.go index 28d65c07c21..006ce48bceb 100644 --- a/backend/plugins/circleci/e2e/pipeline_test.go +++ b/backend/plugins/circleci/e2e/pipeline_test.go @@ -50,7 +50,7 @@ func TestCircleciPipeline(t *testing.T) { e2ehelper.TableOptions{ CSVRelPath: "./snapshot_tables/_tool_circleci_pipelines.csv", IgnoreTypes: []interface{}{common.NoPKModel{}}, - IgnoreFields: []string{"started_at", "stopped_at"}, + IgnoreFields: []string{"stopped_date"}, }, ) } diff --git a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv index dd01015be55..ac203f861b7 100644 --- a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv +++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv @@ -1,11 +1,11 @@ -connection_id,id,project_slug,errors,updated_at,number,trigger_parameters,state,created_at,trigger,vcs,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -1,23622ee4-e150-4920-9d66-81533fa765a4,github/coldgust/coldgust.github.io,[],2023-03-25 17:52:15.651,5,,created,2023-03-25 17:52:15.651,"{""type"":""webhook"",""received_at"":""2023-03-25T17:52:15.446Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""d65e36c3316c6350c0eeb2dcaa6411a49d2aa29d"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,26,"" -1,2c45280f-7fb3-4025-b703-a547c4a94916,github/coldgust/coldgust.github.io,[],2023-03-25 17:50:20.316,4,,created,2023-03-25 17:50:20.316,"{""type"":""webhook"",""received_at"":""2023-03-25T17:50:20.150Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""5d69b9233ed0579b0726caa9d36564cf90a7c63f"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,27,"" -1,625ca634-68fe-4515-91f0-7ba8af51dc99,github/coldgust/coldgust.github.io,[],2023-03-25 17:39:23.233,3,,created,2023-03-25 17:39:23.233,"{""type"":""webhook"",""received_at"":""2023-03-25T17:39:22.921Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""578575831946b53b763ef2807580320f8d8ff12d"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""Add .circleci/config.yml"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,28,"" -1,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,github/coldgust/coldgust.github.io,[],2023-03-25 17:54:08.987,6,,created,2023-03-25 17:54:08.987,"{""type"":""webhook"",""received_at"":""2023-03-25T17:54:08.742Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""fd096b2dbc1ae2ea4c37e7af5101b123e7f60223"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,25,"" -1,7fcc1623-edcc-4a76-ad20-cd81aa83519f,github/coldgust/coldgust.github.io,[],2023-03-25 18:13:21.590,9,,created,2023-03-25 18:13:21.590,"{""type"":""webhook"",""received_at"":""2023-03-25T18:13:21.404Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""49874a6ac988e2b3434501d4a4554c54583e6820"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,22,"" -1,866e967d-f826-4470-aed6-fc0c92e98703,github/coldgust/coldgust.github.io,[],2023-03-25 17:56:24.409,7,,created,2023-03-25 17:56:24.409,"{""type"":""webhook"",""received_at"":""2023-03-25T17:56:24.245Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""ad3aab6479048a72b6aade91b36fdeb7125a16a6"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,24,"" -1,87aad008-1ad5-486a-8174-fdeed846561a,github/coldgust/coldgust.github.io,[],2023-03-25 17:12:18.747,2,,created,2023-03-25 17:12:18.747,"{""type"":""webhook"",""received_at"":""2023-03-25T17:12:18.624Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""9fcceda0e9749bc63e6397b23531fb6bf83634df"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""Add .circleci/config.yml"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,29,"" -1,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,github/coldgust/coldgust.github.io,[],2023-03-25 18:06:13.404,8,,created,2023-03-25 18:06:13.404,"{""type"":""webhook"",""received_at"":""2023-03-25T18:06:13.246Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""7fb6961080f03defe9b7abb35a111a560e6ab1cc"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,23,"" -1,afef32b3-5ffe-48d2-8d9e-46dcedd82554,github/coldgust/coldgust.github.io,[],2023-03-25 17:12:18.398,1,,created,2023-03-25 17:12:18.398,"{""type"":""api"",""received_at"":""2023-03-25T17:12:16.438Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""9fcceda0e9749bc63e6397b23531fb6bf83634df"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":"""",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,30,"" -1,d323f088-02fa-4ed5-9696-fc2f89a27150,github/coldgust/coldgust.github.io,"[{""type"":""config-fetch"",""message"":""Failed to fetch config.yml file.""}]",2023-03-25 18:13:43.446,10,"{""circleci"":{""org_name"":""coldgust"",""project_name"":""coldgust.github.io"",""provider_name"":""github""},""git"":{""checkout_sha"":""6ba0cdaf7e7791c5bc5aa89a13b9cd184f8ad296""}}",errored,2023-03-25 18:13:43.446,"{""type"":""Decoupled Ingestion System"",""received_at"":""1970-01-01T00:00:00.000Z"",""actor"":{""login"":"""",""avatar_url"":""""}}","{""provider_name"":"""",""target_repository_url"":"""",""branch"":"""",""review_id"":"""",""review_url"":"""",""revision"":"""",""tag"":"""",""origin_repository_url"":"""",""commit"":{""subject"":"""",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,21,"" +connection_id,id,project_slug,updated_date,created_date,number,trigger_parameters,state,trigger,vcs +1,23622ee4-e150-4920-9d66-81533fa765a4,github/coldgust/coldgust.github.io,2023-03-25T17:52:15.651+00:00,2023-03-25T17:52:15.651+00:00,5,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:52:15.446Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""d65e36c3316c6350c0eeb2dcaa6411a49d2aa29d"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}" +1,2c45280f-7fb3-4025-b703-a547c4a94916,github/coldgust/coldgust.github.io,2023-03-25T17:50:20.316+00:00,2023-03-25T17:50:20.316+00:00,4,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:50:20.150Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""5d69b9233ed0579b0726caa9d36564cf90a7c63f"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}" +1,625ca634-68fe-4515-91f0-7ba8af51dc99,github/coldgust/coldgust.github.io,2023-03-25T17:39:23.233+00:00,2023-03-25T17:39:23.233+00:00,3,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:39:22.921Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""578575831946b53b763ef2807580320f8d8ff12d"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""Add .circleci/config.yml"",""body"":""""}}" +1,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,github/coldgust/coldgust.github.io,2023-03-25T17:54:08.987+00:00,2023-03-25T17:54:08.987+00:00,6,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:54:08.742Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""fd096b2dbc1ae2ea4c37e7af5101b123e7f60223"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}" +1,7fcc1623-edcc-4a76-ad20-cd81aa83519f,github/coldgust/coldgust.github.io,2023-03-25T18:13:21.590+00:00,2023-03-25T18:13:21.590+00:00,9,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T18:13:21.404Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""49874a6ac988e2b3434501d4a4554c54583e6820"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}" +1,866e967d-f826-4470-aed6-fc0c92e98703,github/coldgust/coldgust.github.io,2023-03-25T17:56:24.409+00:00,2023-03-25T17:56:24.409+00:00,7,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:56:24.245Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""ad3aab6479048a72b6aade91b36fdeb7125a16a6"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}" +1,87aad008-1ad5-486a-8174-fdeed846561a,github/coldgust/coldgust.github.io,2023-03-25T17:12:18.747+00:00,2023-03-25T17:12:18.747+00:00,2,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:12:18.624Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""9fcceda0e9749bc63e6397b23531fb6bf83634df"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""Add .circleci/config.yml"",""body"":""""}}" +1,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,github/coldgust/coldgust.github.io,2023-03-25T18:06:13.404+00:00,2023-03-25T18:06:13.404+00:00,8,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T18:06:13.246Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""7fb6961080f03defe9b7abb35a111a560e6ab1cc"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}" +1,afef32b3-5ffe-48d2-8d9e-46dcedd82554,github/coldgust/coldgust.github.io,2023-03-25T17:12:18.398+00:00,2023-03-25T17:12:18.398+00:00,1,,created,"{""type"":""api"",""received_at"":""2023-03-25T17:12:16.438Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""9fcceda0e9749bc63e6397b23531fb6bf83634df"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":"""",""body"":""""}}" +1,d323f088-02fa-4ed5-9696-fc2f89a27150,github/coldgust/coldgust.github.io,2023-03-25T18:13:43.446+00:00,2023-03-25T18:13:43.446+00:00,10,"{""circleci"":{""org_name"":""coldgust"",""project_name"":""coldgust.github.io"",""provider_name"":""github""},""git"":{""checkout_sha"":""6ba0cdaf7e7791c5bc5aa89a13b9cd184f8ad296""}}",errored,"{""type"":""Decoupled Ingestion System"",""received_at"":""1970-01-01T00:00:00.000Z"",""actor"":{""login"":"""",""avatar_url"":""""}}","{""provider_name"":"""",""target_repository_url"":"""",""branch"":"""",""review_id"":"""",""review_url"":"""",""revision"":"""",""tag"":"""",""origin_repository_url"":"""",""commit"":{""subject"":"""",""body"":""""}}" diff --git a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv index 0c43df3ea4e..2ca84d9a0b0 100644 --- a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv +++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv @@ -1,10 +1,10 @@ -connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status,started_by,pipeline_number,created_at,stopped_at,duration_sec,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -1,6731159f-5275-4bfa-ba70-39d343d63814,github/coldgust/coldgust.github.io,23622ee4-e150-4920-9d66-81533fa765a4,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,5,2023-03-25 17:52:15,2023-03-25 17:52:23,8,2023-03-28 15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,19,"" -1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,github/coldgust/coldgust.github.io,866e967d-f826-4470-aed6-fc0c92e98703,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,7,2023-03-25 17:56:24,2023-03-25 17:56:43,19,2023-03-28 15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,24,"" -1,89054eb2-8e85-4f5c-9a93-66d753a0e970,github/coldgust/coldgust.github.io,625ca634-68fe-4515-91f0-7ba8af51dc99,"",say-hello-workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,3,2023-03-25 17:39:23,2023-03-25 17:39:28,5,2023-03-28 15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,21,"" -1,8971a56b-5547-4824-94dd-07bb467524c5,github/coldgust/coldgust.github.io,87aad008-1ad5-486a-8174-fdeed846561a,"",say-hello-workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,2,2023-03-25 17:12:18,2023-03-25 17:12:23,5,2023-03-28 15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,23,"" -1,8fe60291-68f7-40e2-acec-d99bff4da713,github/coldgust/coldgust.github.io,afef32b3-5ffe-48d2-8d9e-46dcedd82554,"",say-hello-workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,1,2023-03-25 17:12:18,2023-03-25 17:12:22,4,2023-03-28 15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,25,"" -1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,github/coldgust/coldgust.github.io,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,6,2023-03-25 17:54:09,2023-03-25 17:54:23,14,2023-03-28 15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,26,"" -1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,github/coldgust/coldgust.github.io,2c45280f-7fb3-4025-b703-a547c4a94916,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,4,2023-03-25 17:50:20,2023-03-25 17:50:25,5,2023-03-28 15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,20,"" -1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,github/coldgust/coldgust.github.io,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,8,2023-03-25 18:06:13,2023-03-25 18:06:28,15,2023-03-28 15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,27,"" -1,fd0bd4f5-264f-4e3c-a151-06153c018f78,github/coldgust/coldgust.github.io,7fcc1623-edcc-4a76-ad20-cd81aa83519f,"",workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,9,2023-03-25 18:13:21,2023-03-25 18:13:38,17,2023-03-28 15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,22,"" +connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status,started_by,pipeline_number,created_date,stopped_date,duration_sec +1,6731159f-5275-4bfa-ba70-39d343d63814,github/coldgust/coldgust.github.io,23622ee4-e150-4920-9d66-81533fa765a4,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,5,2023-03-25T17:52:15.000+00:00,2023-03-25T17:52:23.000+00:00,8 +1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,github/coldgust/coldgust.github.io,866e967d-f826-4470-aed6-fc0c92e98703,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,7,2023-03-25T17:56:24.000+00:00,2023-03-25T17:56:43.000+00:00,19 +1,89054eb2-8e85-4f5c-9a93-66d753a0e970,github/coldgust/coldgust.github.io,625ca634-68fe-4515-91f0-7ba8af51dc99,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,3,2023-03-25T17:39:23.000+00:00,2023-03-25T17:39:28.000+00:00,5 +1,8971a56b-5547-4824-94dd-07bb467524c5,github/coldgust/coldgust.github.io,87aad008-1ad5-486a-8174-fdeed846561a,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,2,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:23.000+00:00,5 +1,8fe60291-68f7-40e2-acec-d99bff4da713,github/coldgust/coldgust.github.io,afef32b3-5ffe-48d2-8d9e-46dcedd82554,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,1,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:22.000+00:00,4 +1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,github/coldgust/coldgust.github.io,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,6,2023-03-25T17:54:09.000+00:00,2023-03-25T17:54:23.000+00:00,14 +1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,github/coldgust/coldgust.github.io,2c45280f-7fb3-4025-b703-a547c4a94916,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,4,2023-03-25T17:50:20.000+00:00,2023-03-25T17:50:25.000+00:00,5 +1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,github/coldgust/coldgust.github.io,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,8,2023-03-25T18:06:13.000+00:00,2023-03-25T18:06:28.000+00:00,15 +1,fd0bd4f5-264f-4e3c-a151-06153c018f78,github/coldgust/coldgust.github.io,7fcc1623-edcc-4a76-ad20-cd81aa83519f,,workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,9,2023-03-25T18:13:21.000+00:00,2023-03-25T18:13:38.000+00:00,17 From bbd54442ff1f38c2f7a3c53c0ee7b2aa726812b6 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Wed, 17 Jul 2024 21:07:26 +0800 Subject: [PATCH 3/3] fix: migration script versio --- .../migrationscripts/20240717_add_createddate_to_workflow.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/plugins/circleci/models/migrationscripts/20240717_add_createddate_to_workflow.go b/backend/plugins/circleci/models/migrationscripts/20240717_add_createddate_to_workflow.go index b17d3c0ec5e..73f0a12a10f 100644 --- a/backend/plugins/circleci/models/migrationscripts/20240717_add_createddate_to_workflow.go +++ b/backend/plugins/circleci/models/migrationscripts/20240717_add_createddate_to_workflow.go @@ -43,7 +43,7 @@ func (*addCreatedStoppedDateToWorkflow) Up(basicRes context.BasicRes) errors.Err } func (*addCreatedStoppedDateToWorkflow) Version() uint64 { - return 20231129211000 + return 20240717210714 } func (*addCreatedStoppedDateToWorkflow) Name() string {