diff --git a/go.mod b/go.mod index 87c9902056..2fdee353e1 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/deckarep/golang-set v1.8.0 github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 - github.com/devtron-labs/common-lib v0.0.25-0.20240812113340-f14be466613d + github.com/devtron-labs/common-lib v0.16.1-0.20240903114838-d814f4cd60ca github.com/devtron-labs/go-bitbucket v0.9.60-beta github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 github.com/evanphx/json-patch v5.7.0+incompatible @@ -76,6 +76,7 @@ require ( go.uber.org/zap v1.21.0 golang.org/x/crypto v0.25.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 + golang.org/x/mod v0.17.0 golang.org/x/oauth2 v0.21.0 google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d google.golang.org/grpc v1.59.0 @@ -244,7 +245,6 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect diff --git a/go.sum b/go.sum index fb7bccc773..fd4d1398d5 100644 --- a/go.sum +++ b/go.sum @@ -187,8 +187,8 @@ github.com/devtron-labs/argo-workflows/v3 v3.5.10 h1:6rxQOesOzDz6SgQCMDQNHaehsKF github.com/devtron-labs/argo-workflows/v3 v3.5.10/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 h1:2+Q7Jdhpo/uMiaQiZZzAh+ZX7wEJIFuMFG6DEiMuo64= github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8/go.mod h1:702R6WIf5y9UzKGoCGxQ+x3l5Ws+l0fXg2xlCpSGFZI= -github.com/devtron-labs/common-lib v0.0.25-0.20240812113340-f14be466613d h1:+iWXiVOyf9E0bcTia6x2sLFTM7xJc+9Z8q+BfbYr6eM= -github.com/devtron-labs/common-lib v0.0.25-0.20240812113340-f14be466613d/go.mod h1:a7aCClaxYfnyYEENSe1RnkQCeW2AwmCAPYsuvgk0aW0= +github.com/devtron-labs/common-lib v0.16.1-0.20240903114838-d814f4cd60ca h1:cS0cK+aAsMZ9GNcNK3mp9Bej5X/3cRmD/tDLc9/P1GM= +github.com/devtron-labs/common-lib v0.16.1-0.20240903114838-d814f4cd60ca/go.mod h1:a7aCClaxYfnyYEENSe1RnkQCeW2AwmCAPYsuvgk0aW0= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 h1:xwbTeijNTf4/j1v+tSfwVqwLVnReas/NqEKeQHvSTys= diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 08710aac5b..9cf6a18d82 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -105,6 +105,8 @@ type CiArtifactRepository interface { Get(id int) (artifact *CiArtifact, err error) GetArtifactParentCiAndWorkflowDetailsByIds(ids []int) ([]*CiArtifact, error) GetByWfId(wfId int) (artifact *CiArtifact, err error) + IfArtifactExistByImage(imageName string, pipelineId int) (exist bool, err error) + IfArtifactExistByImageDigest(imageDigest string, imageName string, pipelineId int) (exist bool, err error) GetArtifactsByCDPipeline(cdPipelineId, limit int, parentId int, parentType bean.WorkflowType) ([]*CiArtifact, error) GetArtifactsByCDPipelineV3(listingFilterOpts *bean.ArtifactsListFilterOptions) ([]*CiArtifact, int, error) GetLatestArtifactTimeByCiPipelineIds(ciPipelineIds []int) ([]*CiArtifact, error) @@ -316,6 +318,29 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipeline(cdPipelineId, limi return artifactsAll, err } +func (impl CiArtifactRepositoryImpl) IfArtifactExistByImage(imageName string, pipelineId int) (exist bool, err error) { + count, err := impl.dbConnection.Model(&CiArtifact{}). + Where("image = ?", imageName). + Where("pipeline_id = ?", pipelineId). + Count() + if err != nil { + return false, err + } + return count > 0, nil +} + +func (impl CiArtifactRepositoryImpl) IfArtifactExistByImageDigest(imageDigest string, imageName string, pipelineId int) (exist bool, err error) { + count, err := impl.dbConnection.Model(&CiArtifact{}). + Where("image_digest = ?", imageDigest). + Where("image = ?", imageName). + Where("pipeline_id = ?", pipelineId). + Count() + if err != nil { + return false, err + } + return count > 0, nil +} + func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineV3(listingFilterOpts *bean.ArtifactsListFilterOptions) ([]*CiArtifact, int, error) { if listingFilterOpts.ParentStageType != bean.CI_WORKFLOW_TYPE && listingFilterOpts.ParentStageType != bean.WEBHOOK_WORKFLOW_TYPE { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index c960782585..c9e0491e27 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -21,6 +21,8 @@ import ( "errors" "fmt" "github.com/caarlos0/env" + "github.com/devtron-labs/common-lib/utils" + bean3 "github.com/devtron-labs/common-lib/utils/bean" "github.com/devtron-labs/devtron/pkg/infraConfig" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/bean/CiPipeline" @@ -78,6 +80,7 @@ type CiServiceImpl struct { eventClient client.EventClient eventFactory client.EventFactory ciPipelineRepository pipelineConfig.CiPipelineRepository + ciArtifactRepository repository5.CiArtifactRepository pipelineStageService PipelineStageService userService user.UserService ciTemplateService CiTemplateService @@ -99,6 +102,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService ciWorkflowRepository pipelineConfig.CiWorkflowRepository, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, + ciArtifactRepository repository5.CiArtifactRepository, pipelineStageService PipelineStageService, userService user.UserService, ciTemplateService CiTemplateService, appCrudOperationService app.AppCrudOperationService, envRepository repository1.EnvironmentRepository, appRepository appRepository.AppRepository, @@ -122,6 +126,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService eventClient: eventClient, eventFactory: eventFactory, ciPipelineRepository: ciPipelineRepository, + ciArtifactRepository: ciArtifactRepository, pipelineStageService: pipelineStageService, userService: userService, ciTemplateService: ciTemplateService, @@ -158,15 +163,64 @@ func (impl *CiServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelin } } -func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial) error { +func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { + // externalCi artifact is meant only for CI_JOB + if trigger.PipelineType != string(CiPipeline.CI_JOB) { + return nil + } + // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner externalCiArtifact, exists := trigger.ExtraEnvironmentVariables[CiPipeline.ExtraEnvVarExternalCiArtifactKey] // validate externalCiArtifact as docker image if exists { + externalCiArtifact = strings.TrimSpace(externalCiArtifact) if !strings.Contains(externalCiArtifact, ":") { - impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact) - return fmt.Errorf("invalid image name given in externalCiArtifact") + if utils.IsValidDockerTagName(externalCiArtifact) { + fullImageUrl, err := utils.BuildDockerImagePath(bean3.DockerRegistryInfo{ + DockerImageTag: externalCiArtifact, + DockerRegistryId: workflowRequest.DockerRegistryId, + DockerRegistryType: workflowRequest.DockerRegistryType, + DockerRegistryURL: workflowRequest.DockerRegistryURL, + DockerRepository: workflowRequest.DockerRepository, + }) + if err != nil { + impl.Logger.Errorw("Error in building docker image", "err", err) + return err + } + externalCiArtifact = fullImageUrl + } else { + impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact) + return fmt.Errorf("invalid image name or url given in externalCiArtifact") + } + + } + + trigger.ExtraEnvironmentVariables[CiPipeline.ExtraEnvVarExternalCiArtifactKey] = externalCiArtifact + + var artifactExists bool + var err error + if trigger.ExtraEnvironmentVariables[CiPipeline.ExtraEnvVarImageDigestKey] == "" { + artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching ci artifact", "err", err) + return err + } + if artifactExists { + impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact) + return fmt.Errorf("ci artifact already exists with same image name") + } + } else { + artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(trigger.ExtraEnvironmentVariables[CiPipeline.ExtraEnvVarImageDigestKey], externalCiArtifact, trigger.PipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching ci artifact", "err", err) + return err + } + if artifactExists { + impl.Logger.Errorw("ci artifact already exists with same digest", "artifact", externalCiArtifact) + return fmt.Errorf("ci artifact already exists with same digest") + } } + } if trigger.PipelineType == string(CiPipeline.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { ciMaterials[0].GitMaterial = nil @@ -181,10 +235,6 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) if err != nil { return 0, err } - err = impl.handleRuntimeParamsValidations(trigger, ciMaterials) - if err != nil { - return 0, err - } ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) if err != nil && !util.IsErrNoRows(err) { return 0, err @@ -265,6 +315,17 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) impl.Logger.Errorw("make workflow req", "err", err) return 0, err } + err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest) + if err != nil { + savedCiWf.Status = pipelineConfig.WorkflowAborted + savedCiWf.Message = err.Error() + err1 := impl.ciWorkflowRepository.UpdateWorkFlow(savedCiWf) + if err1 != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") + } + return 0, err + } + workflowRequest.Scope = scope workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport diff --git a/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index 671e4502fd..95c0f3b747 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -18,13 +18,20 @@ package utils import ( "fmt" + "github.com/devtron-labs/common-lib/git-manager/util" + "github.com/devtron-labs/common-lib/utils/bean" + "log" "math/rand" + "path" + "regexp" "strings" "time" ) var chars = []rune("abcdefghijklmnopqrstuvwxyz0123456789") +const DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" + // Generates random string func Generate(size int) string { rand.Seed(time.Now().UnixNano()) @@ -47,3 +54,31 @@ func GetUrlWithScheme(url string) (urlWithScheme string) { } return urlWithScheme } + +func IsValidDockerTagName(tagName string) bool { + regString := "^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$" + regexpCompile := regexp.MustCompile(regString) + if regexpCompile.MatchString(tagName) { + return true + } else { + return false + } +} + +func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { + dest := "" + if DOCKER_REGISTRY_TYPE_DOCKERHUB == dockerInfo.DockerRegistryType { + dest = dockerInfo.DockerRepository + ":" + dockerInfo.DockerImageTag + } else { + registryUrl := dockerInfo.DockerRegistryURL + u, err := util.ParseUrl(registryUrl) + if err != nil { + log.Println("not a valid docker repository url") + return "", err + } + u.Path = path.Join(u.Path, "/", dockerInfo.DockerRepository) + dockerRegistryURL := u.Host + u.Path + dest = dockerRegistryURL + ":" + dockerInfo.DockerImageTag + } + return dest, nil +} diff --git a/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index dc32dadbda..07efdf9a56 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -19,3 +19,11 @@ package bean const ( YamlSeparator string = "---\n" ) + +type DockerRegistryInfo struct { + DockerImageTag string `json:"dockerImageTag"` + DockerRegistryId string `json:"dockerRegistryId"` + DockerRegistryType string `json:"dockerRegistryType"` + DockerRegistryURL string `json:"dockerRegistryURL"` + DockerRepository string `json:"dockerRepository"` +} diff --git a/vendor/modules.txt b/vendor/modules.txt index b184ab403e..4bd8d2d94a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -348,7 +348,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.25-0.20240812113340-f14be466613d +# github.com/devtron-labs/common-lib v0.16.1-0.20240903114838-d814f4cd60ca ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage diff --git a/wire_gen.go b/wire_gen.go index 0bc013a9cf..ee50c772b8 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -571,7 +571,7 @@ func InitializeApp() (*App, error) { deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, infraProviderImpl, ciCdPipelineOrchestratorImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, infraProviderImpl, ciCdPipelineOrchestratorImpl) ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, k8sServiceImpl) if err != nil { return nil, err