From ba732570a2f1a64c6dbcbcc925f3b45a61185997 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 22 Aug 2024 00:28:21 +0530 Subject: [PATCH 1/3] SkipCiBuildCachePushPull code incorporated with minor refac in handle runtime params validation --- pkg/pipeline/CiService.go | 29 ++++++++++++------- pkg/pipeline/bean/CiPipeline/CiBuildConfig.go | 5 ++++ pkg/pipeline/types/CiCdConfig.go | 1 + pkg/pipeline/types/Workflow.go | 1 + 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 2d8701d4dd..4a9e9cd154 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -158,27 +158,33 @@ func (impl *CiServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelin } } -func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { - impl.Logger.Debug("ci pipeline manual trigger") - ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) - if err != nil { - return 0, err - } - +func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial) error { // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner - externalCiArtifact, exists := trigger.ExtraEnvironmentVariables["externalCiArtifact"] + externalCiArtifact, exists := trigger.ExtraEnvironmentVariables[CiPipeline.ExtraEnvVarExternalCiArtifactKey] // validate externalCiArtifact as docker image if exists { if !strings.Contains(externalCiArtifact, ":") { impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact) - return 0, fmt.Errorf("invalid image name given in externalCiArtifact") + return fmt.Errorf("invalid image name given in externalCiArtifact") } } if trigger.PipelineType == string(CiPipeline.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { - ciMaterials = []*pipelineConfig.CiPipelineMaterial{ciMaterials[0]} ciMaterials[0].GitMaterial = nil ciMaterials[0].GitMaterialId = 0 } + return nil +} + +func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { + impl.Logger.Debug("ci pipeline manual trigger") + ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) + if err != nil { + return 0, err + } + err = impl.handleRuntimeParamsValidations(trigger, ciMaterials) + if err != nil { + return 0, err + } ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) if err != nil && !util.IsErrNoRows(err) { return 0, err @@ -741,6 +747,9 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. if pipeline.App.AppType == helper.Job { workflowRequest.AppName = pipeline.App.DisplayName } + if trigger.PipelineType == string(CiPipeline.CI_JOB) { + workflowRequest.SkipCiBuildCachePushPull = impl.config.SkipCiBuildCachePushPull + } if dockerRegistry != nil { workflowRequest.DockerRegistryId = dockerRegistry.Id diff --git a/pkg/pipeline/bean/CiPipeline/CiBuildConfig.go b/pkg/pipeline/bean/CiPipeline/CiBuildConfig.go index ef24b43410..0050dc19f3 100644 --- a/pkg/pipeline/bean/CiPipeline/CiBuildConfig.go +++ b/pkg/pipeline/bean/CiPipeline/CiBuildConfig.go @@ -85,3 +85,8 @@ func (pType PipelineType) IsValidPipelineType() bool { return false } } + +const ( + ExtraEnvVarExternalCiArtifactKey = "externalCiArtifact" + ExtraEnvVarImageDigestKey = "imageDigest" +) diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index bf7e82fe3e..4efeb71628 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -82,6 +82,7 @@ type CiCdConfig struct { ImageScanRetryDelay int `env:"IMAGE_SCAN_RETRY_DELAY" envDefault:"5"` ShowDockerBuildCmdInLogs bool `env:"SHOW_DOCKER_BUILD_ARGS" envDefault:"true"` IgnoreCmCsInCiJob bool `env:"IGNORE_CM_CS_IN_CI_JOB" envDefault:"false"` + SkipCiBuildCachePushPull bool `env:"SKIP_CI_BUILD_CACHE_PUSH_PULL" envDefault:"true"` // from CdConfig CdLimitCpu string `env:"CD_LIMIT_CI_CPU" envDefault:"0.5"` CdLimitMem string `env:"CD_LIMIT_CI_MEM" envDefault:"3G"` diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index af15733e1c..e1e578c514 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -144,6 +144,7 @@ type WorkflowRequest struct { Scope resourceQualifiers.Scope BuildxCacheModeMin bool `json:"buildxCacheModeMin"` AsyncBuildxCacheExport bool `json:"asyncBuildxCacheExport"` + SkipCiBuildCachePushPull bool `json:"skipCiBuildCachePushPull"` } func (workflowRequest *WorkflowRequest) updateExternalRunMetadata() { From 02dff3f45ee8ebd6b8376f7aaf49d70873f98649 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 22 Aug 2024 00:45:36 +0530 Subject: [PATCH 2/3] minor refactor --- pkg/pipeline/CiService.go | 2 +- pkg/pipeline/types/CiCdConfig.go | 2 +- pkg/pipeline/types/Workflow.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 4a9e9cd154..613ad50050 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -748,7 +748,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. workflowRequest.AppName = pipeline.App.DisplayName } if trigger.PipelineType == string(CiPipeline.CI_JOB) { - workflowRequest.SkipCiBuildCachePushPull = impl.config.SkipCiBuildCachePushPull + workflowRequest.SkipCiJobBuildCachePushPull = impl.config.SkipCiJobBuildCachePushPull } if dockerRegistry != nil { diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 4efeb71628..a7e23eb97a 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -82,7 +82,7 @@ type CiCdConfig struct { ImageScanRetryDelay int `env:"IMAGE_SCAN_RETRY_DELAY" envDefault:"5"` ShowDockerBuildCmdInLogs bool `env:"SHOW_DOCKER_BUILD_ARGS" envDefault:"true"` IgnoreCmCsInCiJob bool `env:"IGNORE_CM_CS_IN_CI_JOB" envDefault:"false"` - SkipCiBuildCachePushPull bool `env:"SKIP_CI_BUILD_CACHE_PUSH_PULL" envDefault:"true"` + SkipCiJobBuildCachePushPull bool `env:"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL" envDefault:"true"` // from CdConfig CdLimitCpu string `env:"CD_LIMIT_CI_CPU" envDefault:"0.5"` CdLimitMem string `env:"CD_LIMIT_CI_MEM" envDefault:"3G"` diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index e1e578c514..8571885d11 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -144,7 +144,7 @@ type WorkflowRequest struct { Scope resourceQualifiers.Scope BuildxCacheModeMin bool `json:"buildxCacheModeMin"` AsyncBuildxCacheExport bool `json:"asyncBuildxCacheExport"` - SkipCiBuildCachePushPull bool `json:"skipCiBuildCachePushPull"` + SkipCiJobBuildCachePushPull bool `json:"skipCiJobBuildCachePushPull"` } func (workflowRequest *WorkflowRequest) updateExternalRunMetadata() { From a9c99a7d82c8b70fc0dd44bdcd6a2c9a5fcb6010 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 22 Aug 2024 12:17:38 +0530 Subject: [PATCH 3/3] minor refactor --- env_gen.md | 1 + pkg/pipeline/CiService.go | 3 ++- pkg/pipeline/types/CiCdConfig.go | 2 +- pkg/pipeline/types/Workflow.go | 1 - 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/env_gen.md b/env_gen.md index 6e3c3e4f9f..2f4d37a3d2 100644 --- a/env_gen.md +++ b/env_gen.md @@ -231,6 +231,7 @@ | SCOPED_VARIABLE_HANDLE_PRIMITIVES | false | | | SCOPED_VARIABLE_NAME_REGEX | ^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$ | | | SHOW_DOCKER_BUILD_ARGS | true | | + | SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL | false | | | SKIP_CREATING_ECR_REPO | false | | | SOCKET_DISCONNECT_DELAY_SECONDS | 5 | | | SOCKET_HEARTBEAT_SECONDS | 25 | | diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 613ad50050..d16dd55199 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -748,7 +748,8 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. workflowRequest.AppName = pipeline.App.DisplayName } if trigger.PipelineType == string(CiPipeline.CI_JOB) { - workflowRequest.SkipCiJobBuildCachePushPull = impl.config.SkipCiJobBuildCachePushPull + workflowRequest.IgnoreDockerCachePush = impl.config.SkipCiJobBuildCachePushPull + workflowRequest.IgnoreDockerCachePull = impl.config.SkipCiJobBuildCachePushPull } if dockerRegistry != nil { diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index a7e23eb97a..50e7d27271 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -82,7 +82,7 @@ type CiCdConfig struct { ImageScanRetryDelay int `env:"IMAGE_SCAN_RETRY_DELAY" envDefault:"5"` ShowDockerBuildCmdInLogs bool `env:"SHOW_DOCKER_BUILD_ARGS" envDefault:"true"` IgnoreCmCsInCiJob bool `env:"IGNORE_CM_CS_IN_CI_JOB" envDefault:"false"` - SkipCiJobBuildCachePushPull bool `env:"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL" envDefault:"true"` + SkipCiJobBuildCachePushPull bool `env:"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL" envDefault:"false"` // from CdConfig CdLimitCpu string `env:"CD_LIMIT_CI_CPU" envDefault:"0.5"` CdLimitMem string `env:"CD_LIMIT_CI_MEM" envDefault:"3G"` diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index 8571885d11..af15733e1c 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -144,7 +144,6 @@ type WorkflowRequest struct { Scope resourceQualifiers.Scope BuildxCacheModeMin bool `json:"buildxCacheModeMin"` AsyncBuildxCacheExport bool `json:"asyncBuildxCacheExport"` - SkipCiJobBuildCachePushPull bool `json:"skipCiJobBuildCachePushPull"` } func (workflowRequest *WorkflowRequest) updateExternalRunMetadata() {