diff --git a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go index 1ce9827f46..d8705860f1 100644 --- a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go +++ b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go @@ -150,7 +150,7 @@ func (handler PipelineTriggerRestHandlerImpl) OverrideConfig(w http.ResponseWrit triggerContext := bean3.TriggerContext{ Context: ctx, } - mergeResp, helmPackageName, err := handler.cdTriggerService.ManualCdTrigger(triggerContext, &overrideRequest) + mergeResp, helmPackageName, _, err := handler.cdTriggerService.ManualCdTrigger(triggerContext, &overrideRequest) span.End() if err != nil { handler.logger.Errorw("request err, OverrideConfig", "err", err, "payload", overrideRequest) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index a8850d0742..1040788b73 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index eba7ae24e5..b82bfb2a9e 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -27,8 +27,9 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - "github.com/devtron-labs/devtron/pkg/argoApplication/helper" + bean3 "github.com/devtron-labs/devtron/pkg/app/bean" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" + "github.com/devtron-labs/devtron/pkg/argoApplication/helper" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" @@ -775,14 +776,15 @@ func (impl *AppServiceImpl) BuildCDSuccessPayload(appName string, environmentNam } type ValuesOverrideResponse struct { - MergedValues string - ReleaseOverrideJSON string - EnvOverride *bean6.EnvConfigOverride - PipelineStrategy *chartConfig.PipelineStrategy - PipelineOverride *chartConfig.PipelineOverride - Artifact *repository.CiArtifact - Pipeline *pipelineConfig.Pipeline - DeploymentConfig *bean2.DeploymentConfig + MergedValues string + ReleaseOverrideJSON string + EnvOverride *bean6.EnvConfigOverride + PipelineStrategy *chartConfig.PipelineStrategy + PipelineOverride *chartConfig.PipelineOverride + Artifact *repository.CiArtifact + Pipeline *pipelineConfig.Pipeline + DeploymentConfig *bean2.DeploymentConfig + ManifestPushTemplate *bean3.ManifestPushTemplate } func (impl *AppServiceImpl) buildACDContext() (acdContext context.Context, err error) { diff --git a/pkg/deployment/deployedApp/DeployedAppService.go b/pkg/deployment/deployedApp/DeployedAppService.go index 1d3a007555..4765720aa1 100644 --- a/pkg/deployment/deployedApp/DeployedAppService.go +++ b/pkg/deployment/deployedApp/DeployedAppService.go @@ -110,7 +110,7 @@ func (impl *DeployedAppServiceImpl) StopStartApp(ctx context.Context, stopReques Context: ctx, ReferenceId: stopRequest.ReferenceId, } - id, _, err := impl.cdTriggerService.ManualCdTrigger(triggerContext, overrideRequest) + id, _, _, err := impl.cdTriggerService.ManualCdTrigger(triggerContext, overrideRequest) if err != nil { impl.logger.Errorw("error in stopping app", "err", err, "appId", stopRequest.AppId, "envId", stopRequest.EnvironmentId) return 0, err diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go index 746fc83ac6..fa5f04ed6e 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go @@ -21,6 +21,7 @@ import ( bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + bean4 "github.com/devtron-labs/devtron/pkg/app/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" @@ -29,9 +30,9 @@ import ( "time" ) -func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) error { +func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) { request.WorkflowType = bean2.CD_WORKFLOW_TYPE_POST - //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time + // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() triggeredBy := request.TriggeredBy pipeline := request.Pipeline @@ -40,21 +41,23 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) er env, namespace, err := impl.getEnvAndNsIfRunStageInEnv(ctx, request) if err != nil { impl.logger.Errorw("error, getEnvAndNsIfRunStageInEnv", "err", err, "pipeline", pipeline, "stage", request.WorkflowType) - return nil + return nil, nil } request.RunStageInEnvNamespace = namespace + cdWf, runner, err := impl.createStartingWfAndRunner(request, triggeredAt) if err != nil { impl.logger.Errorw("error in creating wf starting and runner entry", "err", err, "request", request) - return err + return nil, err } if cdWf.CiArtifact == nil || cdWf.CiArtifact.Id == 0 { cdWf.CiArtifact, err = impl.ciArtifactRepository.Get(cdWf.CiArtifactId) if err != nil { impl.logger.Errorw("error fetching artifact data", "err", err) - return err + return nil, err } } + // Migration of deprecated DataSource Type if cdWf.CiArtifact.IsMigrationRequired() { migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(cdWf.CiArtifact.Id) @@ -63,24 +66,43 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) er } } + filterEvaluationAudit, err := impl.checkFeasibilityForPostStage(pipeline, &request, env, cdWf, triggeredBy) + if err != nil { + impl.logger.Errorw("error, checkFeasibilityForPostStage", "err", err, "pipeline", pipeline) + return nil, nil + } + envDevploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) - return err + return nil, err + } + + dbErr := impl.createAuditDataForDeploymentWindowBypass(request, runner.Id) + if dbErr != nil { + impl.logger.Errorw("error in creating audit data for deployment window bypass", "runnerId", runner.Id, "err", dbErr) + // skip error for audit data creation + } + + err = impl.handlerFilterEvaluationAudit(filterEvaluationAudit, runner) + if err != nil { + impl.logger.Errorw("error, handlerFilterEvaluationAudit", "err", err) + return nil, err } // custom GitOps repo url validation --> Start err = impl.handleCustomGitOpsRepoValidation(runner, pipeline, envDevploymentConfig, triggeredBy) if err != nil { impl.logger.Errorw("custom GitOps repository validation error, TriggerPreStage", "err", err) - return err + return nil, err } // custom GitOps repo url validation --> Ends + // checking vulnerability for the selected image err = impl.checkVulnerabilityStatusAndFailWfIfNeeded(ctx, cdWf.CiArtifact, pipeline, runner, triggeredBy) if err != nil { impl.logger.Errorw("error, checkVulnerabilityStatusAndFailWfIfNeeded", "err", err, "runner", runner) - return err + return nil, err } cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, envDevploymentConfig, triggeredBy) if err != nil { @@ -97,19 +119,23 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) er runner.Status = cdWorkflow.WorkflowFailed runner.Message = err.Error() _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) - return err + return nil, err } - _, err = impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + _, jobHelmPackagePath, err := impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) if err != nil { impl.logger.Errorw("error in submitting workflow", "err", err, "workflowId", cdStageWorkflowRequest.WorkflowId, "pipeline", pipeline, "env", env) - return err + return nil, err + } + manifestPushTempate, err := impl.getManifestPushTemplateForPostStage(request, envDevploymentConfig, jobHelmPackagePath, cdStageWorkflowRequest, cdWf, runner, pipeline, triggeredBy, triggeredAt) + if err != nil { + impl.logger.Errorw("error in getting manifest push template", "err", err) + return nil, err } - wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), cdWf.Id, bean2.CD_WORKFLOW_TYPE_POST) if err != nil { impl.logger.Errorw("error in getting wfr by workflowId and runnerType", "err", err, "wfId", cdWf.Id) - return err + return nil, err } wfr.ImagePathReservationIds = pluginImagePathReservationIds err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(&wfr) @@ -124,19 +150,19 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) er if evtErr != nil { impl.logger.Errorw("CD trigger event not sent", "error", evtErr) } - //creating cd config history entry + // creating cd config history entry err = impl.prePostCdScriptHistoryService.CreatePrePostCdScriptHistory(pipeline, nil, repository3.POST_CD_TYPE, true, triggeredBy, triggeredAt) if err != nil { impl.logger.Errorw("error in creating post cd script entry", "err", err, "pipeline", pipeline) - return err + return nil, err } - return nil + return manifestPushTempate, nil } -func (impl *TriggerServiceImpl) buildWfRequestErrorHandler(runner *pipelineConfig.CdWorkflowRunner, err error, triggeredBy int32) error { +func (impl *TriggerServiceImpl) buildWfRequestErrorHandler(runner *pipelineConfig.CdWorkflowRunner, err error, triggeredBy int32) (*bean4.ManifestPushTemplate, error) { dbErr := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, err, triggeredBy) if dbErr != nil { impl.logger.Errorw("error while updating current runner status to failed, buildWfRequestErrorHandler", "runner", runner.Id, "err", dbErr, "releaseErr", err) } - return err + return nil, err } diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go new file mode 100644 index 0000000000..d99b813b1d --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package devtronApps + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + bean4 "github.com/devtron-labs/devtron/pkg/app/bean" + "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + bean5 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "time" +) + +func (impl *TriggerServiceImpl) checkFeasibilityForPostStage(pipeline *pipelineConfig.Pipeline, request *bean.TriggerRequest, + env *repository.Environment, cdWf *pipelineConfig.CdWorkflow, triggeredBy int32) (interface{}, error) { + //here return type is interface as ResourceFilterEvaluationAudit is not present in this version + return nil, nil +} + +func (impl *TriggerServiceImpl) getManifestPushTemplateForPostStage(request bean.TriggerRequest, envDevploymentConfig *bean5.DeploymentConfig, + jobHelmPackagePath string, cdStageWorkflowRequest *types.WorkflowRequest, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner, + pipeline *pipelineConfig.Pipeline, triggeredBy int32, triggeredAt time.Time) (*bean4.ManifestPushTemplate, error) { + return nil, nil +} diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index f497db5947..da0ec9a741 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -24,15 +24,15 @@ import ( commonBean "github.com/devtron-labs/common-lib/workflow" bean2 "github.com/devtron-labs/devtron/api/bean" gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" - "github.com/devtron-labs/devtron/internal/sql/constants" + constants2 "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" - bean6 "github.com/devtron-labs/devtron/pkg/attributes/bean" + bean6 "github.com/devtron-labs/devtron/pkg/app/bean" + attributesBean "github.com/devtron-labs/devtron/pkg/attributes/bean" bean4 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/bean/common" - bean7 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" bean5 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" adapter2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" @@ -41,6 +41,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/constants" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" @@ -59,9 +60,9 @@ import ( "time" ) -func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) error { +func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) { request.WorkflowType = bean2.CD_WORKFLOW_TYPE_PRE - //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time + // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() triggeredBy := request.TriggeredBy artifact := request.Artifact @@ -70,26 +71,48 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) err env, namespace, err := impl.getEnvAndNsIfRunStageInEnv(ctx, request) if err != nil { impl.logger.Errorw("error, getEnvAndNsIfRunStageInEnv", "err", err, "pipeline", pipeline, "stage", request.WorkflowType) - return nil + return nil, nil } request.RunStageInEnvNamespace = namespace + + filterEvaluationAudit, err := impl.checkFeasibilityForPreStage(pipeline, &request, env, artifact, triggeredBy) + if err != nil { + impl.logger.Errorw("error, checkFeasibilityForPreStage", "err", err, "pipeline", pipeline) + return nil, nil + } + cdWf, runner, err := impl.createStartingWfAndRunner(request, triggeredAt) if err != nil { impl.logger.Errorw("error in creating wf starting and runner entry", "err", err, "request", request) - return err + return nil, err + } + + // setting triggered as same from runner started on (done for release as cd workflow runners are already created) will be same for other flows as runner are created with time.Now() + triggeredAt = runner.StartedOn + + dbErr := impl.createAuditDataForDeploymentWindowBypass(request, runner.Id) + if dbErr != nil { + impl.logger.Errorw("error in creating audit data for deployment window bypass", "runnerId", runner.Id, "err", dbErr) + // skip error for audit data creation + } + + err = impl.handlerFilterEvaluationAudit(filterEvaluationAudit, runner) + if err != nil { + impl.logger.Errorw("error, handlerFilterEvaluationAudit", "err", err) + return nil, err } envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) - return err + return nil, err } // custom GitOps repo url validation --> Start err = impl.handleCustomGitOpsRepoValidation(runner, pipeline, envDeploymentConfig, triggeredBy) if err != nil { impl.logger.Errorw("custom GitOps repository validation error, TriggerPreStage", "err", err) - return err + return nil, err } // custom GitOps repo url validation --> Ends @@ -97,7 +120,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) err err = impl.checkVulnerabilityStatusAndFailWfIfNeeded(ctx, artifact, pipeline, runner, triggeredBy) if err != nil { impl.logger.Errorw("error, checkVulnerabilityStatusAndFailWfIfNeeded", "err", err, "runner", runner) - return err + return nil, err } _, span := otel.Tracer("orchestrator").Start(ctx, "buildWFRequest") cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, envDeploymentConfig, triggeredBy) @@ -112,7 +135,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) err runner.Status = cdWorkflow.WorkflowFailed runner.Message = err.Error() _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) - return err + return nil, err } else { runner.ImagePathReservationIds = imagePathReservationIds _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) @@ -122,11 +145,20 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) err cdStageWorkflowRequest.Pipeline = pipeline cdStageWorkflowRequest.Env = env cdStageWorkflowRequest.Type = pipelineConfigBean.CD_WORKFLOW_PIPELINE_TYPE - _, err = impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + _, jobHelmPackagePath, err := impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) span.End() + if err != nil { + return nil, err + } + manifestPushTemplate, err := impl.getManifestPushTemplateForPreStage(ctx, envDeploymentConfig, pipeline, artifact, jobHelmPackagePath, cdWf, runner, triggeredBy, triggeredAt, request) + if err != nil { + impl.logger.Errorw("error in getting manifest push template", "err", err) + return nil, err + } + err = impl.sendPreStageNotification(ctx, cdWf, pipeline) if err != nil { - return err + return nil, err } //creating cd config history entry _, span = otel.Tracer("orchestrator").Start(ctx, "prePostCdScriptHistoryService.CreatePrePostCdScriptHistory") @@ -134,12 +166,12 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) err span.End() if err != nil { impl.logger.Errorw("error in creating pre cd script entry", "err", err, "pipeline", pipeline) - return err + return nil, err } - return nil + return manifestPushTemplate, nil } -func (impl *TriggerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int, triggerdBy int32, scanExecutionHistoryId int) error { +func (impl *TriggerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int, triggerdBy int32) error { pipeline, err := impl.pipelineRepository.FindById(cdPipelineId) if err != nil { return err @@ -347,21 +379,23 @@ func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages( } // fetch already saved artifacts to check if they are already present - - savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) - if err != nil { - impl.logger.Errorw("error in fetching artifacts by image path", "err", err) - return nil, err - } - if len(savedCIArtifacts) > 0 { - // if already present in ci artifact, return "image path already in use error" - return nil, pipelineConfigBean.ErrImagePathInUse - } - // reserve all images where data will be - imagePathReservationIds, err := impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) - if err != nil { - impl.logger.Errorw("error in reserving image", "err", err) - return imagePathReservationIds, err + var imagePathReservationIds []int + if len(allDestinationImages) > 0 { + savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) + if err != nil { + impl.logger.Errorw("error in fetching artifacts by image path", "err", err) + return nil, err + } + if len(savedCIArtifacts) > 0 { + // if already present in ci artifact, return "image path already in use error" + return nil, pipelineConfigBean.ErrImagePathInUse + } + // reserve all images where data will be + imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) + if err != nil { + impl.logger.Errorw("error in reserving image", "err", err) + return imagePathReservationIds, err + } } return imagePathReservationIds, nil } @@ -475,7 +509,11 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow AuthMode: gitMaterial.GitProvider.AuthMode, }, } - + cloningModeErr := impl.setCloningModeInCIProjectDetail(&ciProjectDetail, ciPipeline.AppId, m) + if cloningModeErr != nil { + impl.logger.Errorw("could not fetch feature flag value", "err", cloningModeErr) + return nil, cloningModeErr + } if len(ciMaterialCurrent.Modifications) > 0 { ciProjectDetail.CommitHash = ciMaterialCurrent.Modifications[0].Revision ciProjectDetail.Author = ciMaterialCurrent.Modifications[0].Author @@ -486,7 +524,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow return nil, err } ciProjectDetail.CommitTime = commitTime.Format(bean4.LayoutRFC3339) - } else if ciPipeline.PipelineType == string(bean7.CI_JOB) { + } else if ciPipeline.PipelineType == string(constants.CI_JOB) { // This has been done to resolve unmarshalling issue in ci-runner, in case of no commit time(eg- polling container images) ciProjectDetail.CommitTime = time.Time{}.Format(bean4.LayoutRFC3339) } else { @@ -495,7 +533,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow } // set webhook data - if m.Type == constants.SOURCE_TYPE_WEBHOOK && len(ciMaterialCurrent.Modifications) > 0 { + if m.Type == constants2.SOURCE_TYPE_WEBHOOK && len(ciMaterialCurrent.Modifications) > 0 { webhookData := ciMaterialCurrent.Modifications[0].WebhookData ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ Id: webhookData.Id, @@ -516,7 +554,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow var refPluginsData []*pipelineConfigBean.RefPluginObject // if pipeline_stage_steps present for pre-CD or post-CD then no need to add stageYaml to cdWorkflowRequest in that // case add PreDeploySteps and PostDeploySteps to cdWorkflowRequest, this is done for backward compatibility - pipelineStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(cdPipeline.Id, runner.WorkflowType.WorkflowTypeToStageType()) + pipelineStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(cdPipeline.Id, runner.WorkflowType.WorkflowTypeToStageType(), false) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching CD pipeline stage", "cdPipelineId", cdPipeline.Id, "stage ", runner.WorkflowType.WorkflowTypeToStageType(), "err", err) return nil, err @@ -611,7 +649,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow image = ReplaceImageTagWithDigest(image, artifact.ImageDigest) } - host, err := impl.attributeService.GetByKey(bean6.HostUrlKey) + host, err := impl.attributeService.GetByKey(attributesBean.HostUrlKey) if err != nil { impl.logger.Errorw("error in getting hostUrl", "err", err) return nil, err @@ -706,14 +744,14 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow } if ciPipeline != nil && ciPipeline.Id > 0 { - sourceCiPipeline, err := impl.getSourceCiPipelineForArtifact(*ciPipeline) + sourceCiPipeline, err := impl.ciCdPipelineOrchestrator.GetSourceCiPipelineForArtifact(*ciPipeline) if err != nil { impl.logger.Errorw("error in getting source ciPipeline for artifact", "err", err) return nil, err } runtimeParams = runtimeParams.AddSystemVariable(bean.APP_NAME, ciPipeline.App.AppName) cdStageWorkflowRequest.CiPipelineType = sourceCiPipeline.PipelineType - buildRegistryConfig, dbErr := impl.getBuildRegistryConfigForArtifact(*sourceCiPipeline, *artifact) + buildRegistryConfig, dbErr := impl.getBuildRegistryConfigForArtifact(sourceCiPipeline, artifact, cdPipeline.AppId) if dbErr != nil { impl.logger.Errorw("error in getting registry credentials for the artifact", "err", dbErr) return nil, dbErr @@ -865,70 +903,64 @@ getBuildRegistryConfigForArtifact performs the following logic to get Pre/Post C If the ci_pipeline_type type is CI_JOB We will always fetch the registry credentials from the ci_template_override table */ -func (impl *TriggerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeline pipelineConfig.CiPipeline, artifact repository.CiArtifact) (*types.DockerArtifactStoreBean, error) { - // Handling for Skopeo Plugin - if artifact.IsRegistryCredentialMapped() { - dockerArtifactStore, err := impl.dockerArtifactStoreRepository.FindOne(artifact.CredentialSourceValue) - if util.IsErrNoRows(err) { - impl.logger.Errorw("source artifact registry not found", "registryId", artifact.CredentialSourceValue, "err", err) - return nil, fmt.Errorf("source artifact registry '%s' not found", artifact.CredentialSourceValue) - } else if err != nil { - impl.logger.Errorw("error in fetching artifact info", "err", err) - return nil, err +func (impl *TriggerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact, appId int) (*types.DockerArtifactStoreBean, error) { + var buildRegistryConfig *types.DockerArtifactStoreBean + var err error + + if sourceCiPipeline != nil && sourceCiPipeline.Id != 0 { + // Handling for Skopeo Plugin + if artifact.IsRegistryCredentialMapped() { + dockerArtifactStore, err := impl.dockerArtifactStoreRepository.FindOne(artifact.CredentialSourceValue) + if util.IsErrNoRows(err) { + impl.logger.Errorw("source artifact registry not found", "registryId", artifact.CredentialSourceValue, "err", err) + return nil, fmt.Errorf("source artifact registry '%s' not found", artifact.CredentialSourceValue) + } else if err != nil { + impl.logger.Errorw("error in fetching artifact info", "err", err) + return nil, err + } + return adapter.GetDockerConfigBean(dockerArtifactStore), nil } - return adapter.GetDockerConfigBean(dockerArtifactStore), nil - } - // Handling for CI Job - if adapter.IsCIJob(sourceCiPipeline) { - // for bean.CI_JOB the source artifact is always driven from overridden ci template - buildRegistryConfig, err := impl.ciTemplateService.GetAppliedDockerConfigForCiPipeline(sourceCiPipeline.Id, sourceCiPipeline.AppId, true) - if err != nil { - impl.logger.Errorw("error in getting build configurations", "err", err) - return nil, fmt.Errorf("error in getting build configurations") + // Handling for CI Job + if adapter.IsCIJob(sourceCiPipeline) { + // for bean.CI_JOB the source artifact is always driven from overridden ci template + buildRegistryConfig, err = impl.ciTemplateService.GetAppliedDockerConfigForCiPipeline(sourceCiPipeline.Id, sourceCiPipeline.AppId, true) + if err != nil { + impl.logger.Errorw("error in getting build configurations", "err", err) + return nil, fmt.Errorf("error in getting build configurations") + } + return buildRegistryConfig, nil } - return buildRegistryConfig, nil - } - // Handling for Linked CI - if adapter.IsLinkedCI(sourceCiPipeline) { - parentCiPipeline, err := impl.ciPipelineRepository.FindById(sourceCiPipeline.ParentCiPipeline) - if err != nil { - impl.logger.Errorw("error in finding ciPipeline", "ciPipelineId", sourceCiPipeline.ParentCiPipeline, "err", err) - return nil, err + // Handling for Linked CI + if adapter.IsLinkedCI(sourceCiPipeline) { + parentCiPipeline, err := impl.ciPipelineRepository.FindById(sourceCiPipeline.ParentCiPipeline) + if err != nil { + impl.logger.Errorw("error in finding ciPipeline", "ciPipelineId", sourceCiPipeline.ParentCiPipeline, "err", err) + return nil, err + } + buildRegistryConfig, err = impl.ciTemplateService.GetAppliedDockerConfigForCiPipeline(parentCiPipeline.Id, parentCiPipeline.AppId, parentCiPipeline.IsDockerConfigOverridden) + if err != nil { + impl.logger.Errorw("error in getting build configurations", "err", err) + return nil, fmt.Errorf("error in getting build configurations") + } + return buildRegistryConfig, nil } - buildRegistryConfig, err := impl.ciTemplateService.GetAppliedDockerConfigForCiPipeline(parentCiPipeline.Id, parentCiPipeline.AppId, parentCiPipeline.IsDockerConfigOverridden) + + // Handling for Build CI + buildRegistryConfig, err = impl.ciTemplateService.GetAppliedDockerConfigForCiPipeline(sourceCiPipeline.Id, sourceCiPipeline.AppId, sourceCiPipeline.IsDockerConfigOverridden) if err != nil { impl.logger.Errorw("error in getting build configurations", "err", err) return nil, fmt.Errorf("error in getting build configurations") } - return buildRegistryConfig, nil - } - - // Handling for Build CI - buildRegistryConfig, err := impl.ciTemplateService.GetAppliedDockerConfigForCiPipeline(sourceCiPipeline.Id, sourceCiPipeline.AppId, sourceCiPipeline.IsDockerConfigOverridden) - if err != nil { - impl.logger.Errorw("error in getting build configurations", "err", err) - return nil, fmt.Errorf("error in getting build configurations") - } - return buildRegistryConfig, nil -} - -func (impl *TriggerServiceImpl) getSourceCiPipelineForArtifact(ciPipeline pipelineConfig.CiPipeline) (*pipelineConfig.CiPipeline, error) { - sourceCiPipeline := &ciPipeline - if adapter.IsLinkedCD(ciPipeline) { - sourceCdPipeline, err := impl.pipelineRepository.FindById(ciPipeline.ParentCiPipeline) + } else { + buildRegistryConfig, err = impl.getPreStageBuildRegistryConfigIfSourcePipelineNotPresent(appId) if err != nil { - impl.logger.Errorw("error in finding source cdPipeline for linked cd", "cdPipelineId", ciPipeline.ParentCiPipeline, "err", err) - return nil, err - } - sourceCiPipeline, err = impl.ciPipelineRepository.FindOneWithAppData(sourceCdPipeline.CiPipelineId) - if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("error in finding ciPipeline for the cd pipeline", "CiPipelineId", sourceCdPipeline.Id, "CiPipelineId", sourceCdPipeline.CiPipelineId, "err", err) + impl.logger.Errorw("error in getting build configuration", "err", err) return nil, err } } - return sourceCiPipeline, nil + return buildRegistryConfig, nil } func (impl *TriggerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go new file mode 100644 index 0000000000..8f5b3a5c39 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go @@ -0,0 +1,47 @@ +package devtronApps + +import ( + "context" + "fmt" + repository2 "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + bean6 "github.com/devtron-labs/devtron/pkg/app/bean" + "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "time" +) + +func (impl *TriggerServiceImpl) checkFeasibilityForPreStage(pipeline *pipelineConfig.Pipeline, request *bean2.TriggerRequest, + env *repository.Environment, artifact *repository2.CiArtifact, triggeredBy int32) (interface{}, error) { + //here return type is interface as ResourceFilterEvaluationAudit is not present in this version + return nil, nil +} + +func (impl *TriggerServiceImpl) createAuditDataForDeploymentWindowBypass(request bean2.TriggerRequest, wfrId int) error { + return nil +} + +func (impl *TriggerServiceImpl) getManifestPushTemplateForPreStage(ctx context.Context, envDeploymentConfig *bean3.DeploymentConfig, + pipeline *pipelineConfig.Pipeline, artifact *repository2.CiArtifact, jobHelmPackagePath string, + cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner, triggeredBy int32, triggeredAt time.Time, + request bean2.TriggerRequest) (*bean6.ManifestPushTemplate, error) { + return nil, nil +} + +func (impl *TriggerServiceImpl) setCloningModeInCIProjectDetail(ciProjectDetail *bean.CiProjectDetails, appId int, + m *pipelineConfig.CiPipelineMaterial) error { + return nil +} + +func (impl *TriggerServiceImpl) getPreStageBuildRegistryConfigIfSourcePipelineNotPresent(appId int) (*types.DockerArtifactStoreBean, error) { + return nil, fmt.Errorf("soucePipeline is mandatory, corrupt data") +} + +func (impl *TriggerServiceImpl) handlerFilterEvaluationAudit(filterEvaluationAudit interface{}, + runner *pipelineConfig.CdWorkflowRunner) error { + //here ip type of filterEvaluationAudit is interface as ResourceFilterEvaluationAudit is not present in this version + return nil +} diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 838b9abe7d..423ce89b94 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -22,10 +22,8 @@ import ( "fmt" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" util5 "github.com/devtron-labs/common-lib/utils/k8s" - "github.com/devtron-labs/common-lib/utils/k8s/commonBean" bean3 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/api/bean/gitOps" - bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" client2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/client/argocdServer" @@ -54,6 +52,7 @@ import ( pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" bean9 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" @@ -102,17 +101,17 @@ import ( ) type TriggerService interface { - TriggerPostStage(request bean.TriggerRequest) error - TriggerPreStage(request bean.TriggerRequest) error + TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) + TriggerPreStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) - TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int, triggerdBy int32, scanExecutionHistoryId int) error + TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int, triggerdBy int32) error TriggerStageForBulk(triggerRequest bean.TriggerRequest) error - ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, error) + ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) TriggerAutomaticDeployment(request bean.TriggerRequest) error - TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, err error) + TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) } type TriggerServiceImpl struct { @@ -174,6 +173,7 @@ type TriggerServiceImpl struct { ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator gitOperationService git.GitOperationService attributeService attributes.AttributesService + clusterRepository repository5.ClusterRepository } func NewTriggerServiceImpl(logger *zap.SugaredLogger, @@ -232,6 +232,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator, gitOperationService git.GitOperationService, attributeService attributes.AttributesService, + clusterRepository repository5.ClusterRepository, ) (*TriggerServiceImpl, error) { impl := &TriggerServiceImpl{ logger: logger, @@ -295,6 +296,8 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, gitOperationService: gitOperationService, attributeService: attributeService, + + clusterRepository: clusterRepository, } config, err := types.GetCdConfig() if err != nil { @@ -306,13 +309,13 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, func (impl *TriggerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { - preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD) + preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD, false) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching CD pipeline stage", "cdPipelineId", triggerRequest.Pipeline.Id, "stage ", repository.PIPELINE_STAGE_TYPE_PRE_CD, "err", err) return err } - //handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) + // handle corrupt data (https://github.com/devtron-labs/devtron/issues/3826) err, deleted := impl.deleteCorruptedPipelineStage(preStage, triggerRequest.TriggeredBy) if err != nil { impl.logger.Errorw("error in deleteCorruptedPipelineStage ", "cdPipelineId", triggerRequest.Pipeline.Id, "err", err, "preStage", preStage, "triggeredBy", triggerRequest.TriggeredBy) @@ -321,10 +324,14 @@ func (impl *TriggerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerR triggerRequest.TriggerContext.Context = context.Background() if len(triggerRequest.Pipeline.PreStageConfig) > 0 || (preStage != nil && !deleted) { - //pre stage exists + // pre stage exists impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", triggerRequest.Artifact.Id, "pipelineId", triggerRequest.Pipeline.Id) triggerRequest.RefCdWorkflowRunnerId = 0 - err = impl.TriggerPreStage(triggerRequest) // TODO handle error here + err = impl.preStageHandlingForTriggerStageInBulk(&triggerRequest) + if err != nil { + return err + } + _, err = impl.TriggerPreStage(triggerRequest) // TODO handle error here return err } else { // trigger deployment @@ -386,11 +393,12 @@ func (impl *TriggerServiceImpl) validateDeploymentTriggerRequest(ctx context.Con } // TODO: write a wrapper to handle auto and manual trigger -func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, error) { +func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { triggerContext.TriggerType = bean.Manual - //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time + // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() + releaseId := 0 ctx := triggerContext.Context cdPipeline, err := impl.getCdPipelineForManualCdTrigger(ctx, overrideRequest.PipelineId) @@ -401,13 +409,14 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte impl.logger.Errorw("error while updating current runner status to failed, ManualCdTrigger", "cdWfr", overrideRequest.WfrId, "err2", err2) } } - return 0, "", err + return 0, "", nil, err } envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(cdPipeline.AppId, cdPipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", cdPipeline.AppId, "envId", cdPipeline.EnvironmentId, "err", err) - return 0, "", err + return 0, "", nil, err } + adapter.SetPipelineFieldsInOverrideRequest(overrideRequest, cdPipeline, envDeploymentConfig) ciArtifactId := overrideRequest.CiArtifactId @@ -416,11 +425,11 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte span.End() if err != nil { impl.logger.Errorw("error in getting CiArtifact", "CiArtifactId", overrideRequest.CiArtifactId, "err", err) - return 0, "", err + return 0, "", nil, err } - // Migration of deprecated DataSource Type if artifact.IsMigrationRequired() { + // Migration of deprecated DataSource Type migrationErr := impl.ciArtifactRepository.MigrateToWebHookDataSourceType(artifact.Id) if migrationErr != nil { impl.logger.Warnw("unable to migrate deprecated DataSource", "artifactId", artifact.Id) @@ -432,6 +441,7 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte impl.logger.Errorw("error in getting image tag and repo", "err", err) } helmPackageName := fmt.Sprintf("%s-%s-%s", cdPipeline.App.AppName, cdPipeline.Environment.Name, imageTag) + var manifestPushTemplate *bean4.ManifestPushTemplate switch overrideRequest.CdWorkflowType { case bean3.CD_WORKFLOW_TYPE_PRE: @@ -444,13 +454,13 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte } err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) if err != nil { - return 0, "", err + return 0, "", nil, err } } else { cdWf, err = impl.cdWorkflowRepository.FindById(overrideRequest.CdWorkflowId) if err != nil { impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) - return 0, "", err + return 0, "", nil, err } } overrideRequest.CdWorkflowId = cdWf.Id @@ -466,11 +476,11 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte RefCdWorkflowRunnerId: 0, CdWorkflowRunnerId: overrideRequest.WfrId, } - err = impl.TriggerPreStage(triggerRequest) + manifestPushTemplate, err = impl.TriggerPreStage(triggerRequest) span.End() if err != nil { impl.logger.Errorw("error in TriggerPreStage, ManualCdTrigger", "err", err) - return 0, "", err + return 0, "", nil, err } case bean3.CD_WORKFLOW_TYPE_DEPLOY: if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { @@ -480,7 +490,7 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte cdWf, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_PRE) if err != nil && !util.IsErrNoRows(err) { impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", err + return 0, "", nil, err } cdWorkflowId := cdWf.CdWorkflowId @@ -493,7 +503,7 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) if err != nil { impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "PipelineId", overrideRequest.PipelineId, "err", err) - return 0, "", err + return 0, "", nil, err } cdWorkflowId = cdWf.Id } @@ -513,7 +523,7 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte savedWfr, err := impl.cdWorkflowRepository.SaveWorkFlowRunner(runner) if err != nil { impl.logger.Errorw("err in creating cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", cdWorkflowId, "err", err) - return 0, "", err + return 0, "", nil, err } runner.CdWorkflow = &pipelineConfig.CdWorkflow{ Pipeline: cdPipeline, @@ -533,26 +543,26 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte validationErr := impl.validateDeploymentTriggerRequest(ctx, runner, cdPipeline, artifact.ImageDigest, envDeploymentConfig, overrideRequest.UserId) if validationErr != nil { impl.logger.Errorw("validation error deployment request", "cdWfr", runner.Id, "err", validationErr) - return 0, "", validationErr + return 0, "", nil, validationErr } } // Deploy the release var releaseErr error - releaseId, releaseErr = impl.handleCDTriggerRelease(ctx, overrideRequest, envDeploymentConfig, triggeredAt, overrideRequest.UserId) + releaseId, manifestPushTemplate, releaseErr = impl.handleCDTriggerRelease(ctx, overrideRequest, envDeploymentConfig, triggeredAt, overrideRequest.UserId) // if releaseErr found, then the mark current deployment Failed and return if releaseErr != nil { err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, overrideRequest.UserId) if err != nil { impl.logger.Errorw("error while updating current runner status to failed", "cdWfr", runner.Id, "err", err) } - return 0, "", releaseErr + return 0, "", nil, releaseErr } case bean3.CD_WORKFLOW_TYPE_POST: cdWfRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, overrideRequest.CdWorkflowId, bean3.CD_WORKFLOW_TYPE_DEPLOY) if err != nil && !util.IsErrNoRows(err) { impl.logger.Errorw("err in getting cdWorkflowRunner, ManualCdTrigger", "cdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", err + return 0, "", nil, err } var cdWf *pipelineConfig.CdWorkflow @@ -565,7 +575,7 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte err := impl.cdWorkflowRepository.SaveWorkFlow(ctx, cdWf) if err != nil { impl.logger.Errorw("error in creating cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", err + return 0, "", nil, err } overrideRequest.CdWorkflowId = cdWf.Id } else { @@ -574,7 +584,7 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte span.End() if err != nil && !util.IsErrNoRows(err) { impl.logger.Errorw("error in getting cdWorkflow, ManualCdTrigger", "CdWorkflowId", overrideRequest.CdWorkflowId, "err", err) - return 0, "", err + return 0, "", nil, err } } _, span = otel.Tracer("orchestrator").Start(ctx, "TriggerPostStage") @@ -586,17 +596,17 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte TriggerContext: triggerContext, CdWorkflowRunnerId: overrideRequest.WfrId, } - err = impl.TriggerPostStage(triggerRequest) + manifestPushTemplate, err = impl.TriggerPostStage(triggerRequest) span.End() if err != nil { impl.logger.Errorw("error in TriggerPostStage, ManualCdTrigger", "CdWorkflowId", cdWf.Id, "err", err) - return 0, "", err + return 0, "", nil, err } default: impl.logger.Errorw("invalid CdWorkflowType, ManualCdTrigger", "CdWorkflowType", overrideRequest.CdWorkflowType, "err", err) - return 0, "", fmt.Errorf("invalid CdWorkflowType %s for the trigger request", string(overrideRequest.CdWorkflowType)) + return 0, "", nil, fmt.Errorf("invalid CdWorkflowType %s for the trigger request", string(overrideRequest.CdWorkflowType)) } - return releaseId, helmPackageName, err + return releaseId, helmPackageName, manifestPushTemplate, err } func isNotHibernateRequest(deploymentType models.DeploymentType) bool { @@ -605,7 +615,7 @@ func isNotHibernateRequest(deploymentType models.DeploymentType) bool { // TODO: write a wrapper to handle auto and manual trigger func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { - //in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here + // in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here triggeredBy := request.TriggeredBy pipeline := request.Pipeline artifact := request.Artifact @@ -744,7 +754,7 @@ func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *p return err } // setting deployedBy as 1(system user) since case of auto trigger - id, err := impl.handleCDTriggerRelease(releaseCtx, request, envDeploymentConfig, triggeredAt, 1) + id, _, err := impl.handleCDTriggerRelease(releaseCtx, request, envDeploymentConfig, triggeredAt, 1) if err != nil { impl.logger.Errorw("error in auto cd pipeline trigger", "pipelineId", pipeline.Id, "artifactId", artifact.Id, "err", err) } else { @@ -753,7 +763,7 @@ func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *p return err } -func (impl *TriggerServiceImpl) triggerAsyncRelease(userDeploymentRequestId int, overrideRequest *bean3.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, err error) { +func (impl *TriggerServiceImpl) triggerAsyncRelease(userDeploymentRequestId int, overrideRequest *bean3.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.triggerAsyncRelease") defer span.End() // build merged values and save PCO history for the release @@ -762,18 +772,18 @@ func (impl *TriggerServiceImpl) triggerAsyncRelease(userDeploymentRequestId int, historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, deployedBy) if historyErr != nil { impl.logger.Errorw("error in auditing deployment trigger history", "cdWfrId", overrideRequest.WfrId, "err", err) - return releaseNo, err + return releaseNo, manifestPushTemplate, err } // handling GetValuesOverrideForTrigger error if err != nil { impl.logger.Errorw("error in fetching values for trigger", "err", err) - return releaseNo, err + return releaseNo, manifestPushTemplate, err } // asynchronous mode of Helm/ArgoCd installation starts return impl.workflowEventPublishService.TriggerAsyncRelease(userDeploymentRequestId, overrideRequest, valuesOverrideResponse, newCtx, deployedBy) } -func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, err error) { +func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.handleCDTriggerRelease") defer span.End() // Handling for auto trigger @@ -783,7 +793,7 @@ func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, over tx, err := impl.transactionUtilImpl.StartTx() if err != nil { impl.logger.Errorw("error in starting transaction to update userDeploymentRequest", "error", err) - return releaseNo, err + return releaseNo, manifestPushTemplate, err } defer impl.transactionUtilImpl.RollbackTx(tx) newDeploymentRequest := adapter.NewUserDeploymentRequest(overrideRequest, triggeredAt, overrideRequest.UserId) @@ -791,7 +801,7 @@ func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, over userDeploymentRequestId, err := impl.userDeploymentRequestService.SaveNewDeployment(newCtx, tx, newDeploymentRequest) if err != nil { impl.logger.Errorw("error in saving new userDeploymentRequest", "overrideRequest", overrideRequest, "err", err) - return releaseNo, err + return releaseNo, manifestPushTemplate, err } timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(overrideRequest.WfrId, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_REQUEST_VALIDATED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_REQUEST_VALIDATED, deployedBy) // creating cd pipeline status timeline for deployment trigger request validated @@ -799,10 +809,15 @@ func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, over err = impl.transactionUtilImpl.CommitTx(tx) if err != nil { impl.logger.Errorw("error in committing transaction to update userDeploymentRequest", "error", err) - return userDeploymentRequestId, err + return userDeploymentRequestId, manifestPushTemplate, err } - - if impl.isDevtronAsyncInstallModeEnabled(overrideRequest.DeploymentAppType, overrideRequest.ForceSyncDeployment) { + isAsyncMode, err := impl.isDevtronAsyncInstallModeEnabled(overrideRequest) + if err != nil { + impl.logger.Errorw("error in checking async mode for devtron app", "err", err, "deploymentType", overrideRequest.DeploymentType, + "forceSyncDeployment", overrideRequest.ForceSyncDeployment, "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId) + return userDeploymentRequestId, manifestPushTemplate, err + } + if isAsyncMode { return impl.triggerAsyncRelease(userDeploymentRequestId, overrideRequest, newCtx, triggeredAt, deployedBy) } // synchronous mode of installation starts @@ -823,18 +838,18 @@ func (impl *TriggerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, value } // TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously -func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, err error) { +func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.TriggerRelease") defer span.End() triggerEvent, skipRequest, err := impl.buildTriggerEventForOverrideRequest(overrideRequest, triggeredAt) if err != nil { - return releaseNo, err + return releaseNo, manifestPushTemplate, err } impl.logger.Debugw("processing TriggerRelease", "wfrId", overrideRequest.WfrId, "triggerEvent", triggerEvent) // request has already been served, skipping if skipRequest { impl.logger.Infow("request already served, skipping", "wfrId", overrideRequest.WfrId) - return releaseNo, nil + return releaseNo, manifestPushTemplate, nil } // build merged values and save PCO history for the release valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(overrideRequest, triggeredAt, newCtx) @@ -843,7 +858,7 @@ func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOver envDeploymentConfig, err1 := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(overrideRequest.AppId, overrideRequest.EnvId) if err1 != nil { impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", err1) - return releaseNo, err1 + return releaseNo, manifestPushTemplate, err1 } valuesOverrideResponse.DeploymentConfig = envDeploymentConfig } @@ -853,16 +868,28 @@ func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOver historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, triggeredBy) if historyErr != nil { impl.logger.Errorw("error in auditing deployment trigger history", "cdWfrId", overrideRequest.WfrId, "err", err) - return releaseNo, err + return releaseNo, manifestPushTemplate, err } if err != nil { impl.logger.Errorw("error in building merged manifest for trigger", "err", err) - return releaseNo, err + impl.manifestGenerationFailedTimelineHandling(triggerEvent, overrideRequest, err) + return releaseNo, manifestPushTemplate, err + } + helmManifest, err := impl.getHelmManifestForTriggerRelease(ctx, triggerEvent, overrideRequest, valuesOverrideResponse, builtChartPath) + if err != nil { + impl.logger.Errorw("error, getHelmManifestForTriggerRelease", "err", err) + return releaseNo, manifestPushTemplate, err } impl.logger.Debugw("triggering pipeline for release", "wfrId", overrideRequest.WfrId, "builtChartPath", builtChartPath) releaseNo, err = impl.triggerPipeline(overrideRequest, valuesOverrideResponse, builtChartPath, triggerEvent, newCtx) if err != nil { - return 0, err + return 0, manifestPushTemplate, err + } + + err = impl.triggerReleaseSuccessHandling(triggerEvent, overrideRequest, valuesOverrideResponse, helmManifest) + if err != nil { + impl.logger.Errorw("error, triggerReleaseSuccessHandling", "triggerEvent", triggerEvent, "err", err) + return releaseNo, manifestPushTemplate, err } // creating cd pipeline status timeline for deployment triggered - for successfully triggered requests timeline := impl.pipelineStatusTimelineService.NewDevtronAppPipelineStatusTimelineDbObject(overrideRequest.WfrId, timelineStatus.TIMELINE_STATUS_DEPLOYMENT_TRIGGERED, timelineStatus.TIMELINE_DESCRIPTION_DEPLOYMENT_COMPLETED, overrideRequest.UserId) @@ -871,7 +898,7 @@ func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOver impl.logger.Errorw("error in creating timeline status for deployment completed", "err", dbErr, "timeline", timeline) } impl.logger.Debugw("triggered pipeline for release successfully", "wfrId", overrideRequest.WfrId, "builtChartPath", builtChartPath) - return releaseNo, nil + return releaseNo, valuesOverrideResponse.ManifestPushTemplate, nil } func (impl *TriggerServiceImpl) performGitOps(ctx context.Context, @@ -900,6 +927,7 @@ func (impl *TriggerServiceImpl) performGitOps(ctx context.Context, // Update GitOps repo url after repo new repo created valuesOverrideResponse.DeploymentConfig.RepoURL = manifestPushResponse.NewGitRepoUrl } + valuesOverrideResponse.ManifestPushTemplate = manifestPushTemplate return nil } @@ -950,6 +978,7 @@ func (impl *TriggerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOve } impl.logger.Debugw("chart push operation completed successfully", "cdWfrId", overrideRequest.WfrId) } + if triggerEvent.PerformDeploymentOnCluster { err = impl.deployApp(newCtx, overrideRequest, valuesOverrideResponse, triggerEvent) if err != nil { @@ -996,10 +1025,15 @@ func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3 return manifestPushTemplate, err } - if manifestPushConfig != nil { + if manifestPushConfig != nil && manifestPushConfig.Id != 0 { if manifestPushConfig.StorageType == bean2.ManifestStorageGit { // need to implement for git repo push // currently manifest push config doesn't have git push config. GitOps config is derived from charts, chart_env_config_override and chart_ref table + } else { + err2 := impl.buildManifestPushTemplateForNonGitStorageType(overrideRequest, valuesOverrideResponse, builtChartPath, err, manifestPushConfig, manifestPushTemplate) + if err2 != nil { + return manifestPushTemplate, err2 + } } } else { manifestPushTemplate.ChartReferenceTemplate = valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate @@ -1009,15 +1043,7 @@ func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3 manifestPushTemplate.RepoUrl = valuesOverrideResponse.DeploymentConfig.RepoURL manifestPushTemplate.IsCustomGitRepository = common.IsCustomGitOpsRepo(valuesOverrideResponse.DeploymentConfig.ConfigType) } - return manifestPushTemplate, err -} - -func (impl *TriggerServiceImpl) getManifestPushService(storageType string) publish.ManifestPushService { - var manifestPushService publish.ManifestPushService - if storageType == bean2.ManifestStorageGit { - manifestPushService = impl.gitOpsManifestPushService - } - return manifestPushService + return manifestPushTemplate, nil } func (impl *TriggerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { @@ -1043,83 +1069,35 @@ func (impl *TriggerServiceImpl) deployApp(ctx context.Context, overrideRequest * return nil } -func (impl *TriggerServiceImpl) postDeployHook(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, referenceChartByte []byte, err error) { - impl.logger.Debugw("no post deploy hook registered") -} - func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.createHelmAppForCdPipeline") defer span.End() pipelineModel := valuesOverrideResponse.Pipeline envOverride := valuesOverrideResponse.EnvOverride mergeAndSave := valuesOverrideResponse.MergedValues - - chartMetaData := &chart.Metadata{ - Name: pipelineModel.App.AppName, - Version: envOverride.Chart.ChartVersion, + chartMetaData, helmRevisionHistory, releaseIdentifier, err := impl.getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx, valuesOverrideResponse) + if err != nil { + impl.logger.Errorw("error, getHelmHistoryLimitAndChartMetadataForHelmAppCreation", "valuesOverrideResponse", valuesOverrideResponse, "err", err) + return false, nil, err } referenceTemplate := envOverride.Chart.ReferenceTemplate referenceTemplatePath := path.Join(bean5.RefChartDirPath, referenceTemplate) var referenceChartByte []byte if util.IsHelmApp(valuesOverrideResponse.DeploymentConfig.DeploymentAppType) { - var sanitizedK8sVersion string - //handle specific case for all cronjob charts from cronjob-chart_1-2-0 to cronjob-chart_1-5-0 where semverCompare - //comparison func has wrong api version mentioned, so for already installed charts via these charts that comparison - //is always false, handles the gh issue:- https://github.com/devtron-labs/devtron/issues/4860 - cronJobChartRegex := regexp.MustCompile(bean.CronJobChartRegexExpression) - if cronJobChartRegex.MatchString(referenceTemplate) { - k8sServerVersion, err := impl.K8sUtil.GetKubeVersion() - if err != nil { - impl.logger.Errorw("exception caught in getting k8sServerVersion", "err", err) - return false, nil, err - } - sanitizedK8sVersion = k8s2.StripPrereleaseFromK8sVersion(k8sServerVersion.String()) - } - referenceChartByte = envOverride.Chart.ReferenceChart - // here updating reference chart into database. - if len(envOverride.Chart.ReferenceChart) == 0 { - refChartByte, err := impl.chartTemplateService.GetByteArrayRefChart(chartMetaData, referenceTemplatePath) - if err != nil { - impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) - return false, nil, err - } - ch := envOverride.Chart - ch.ReferenceChart = refChartByte - ch.UpdatedOn = time.Now() - ch.UpdatedBy = overrideRequest.UserId - err = impl.chartRepository.Update(ch) - if err != nil { - impl.logger.Errorw("chart update error", "err", err, "req", overrideRequest) - return false, nil, err - } - referenceChartByte = refChartByte - } - - releaseName := pipelineModel.DeploymentAppName - cluster := envOverride.Environment.Cluster - bearerToken := cluster.Config[commonBean.BearerToken] - clusterConfig := &gRPC.ClusterConfig{ - ClusterName: cluster.ClusterName, - Token: bearerToken, - ApiServerUrl: cluster.ServerUrl, - InsecureSkipTLSVerify: cluster.InsecureSkipTlsVerify, - } - if cluster.InsecureSkipTlsVerify == false { - clusterConfig.KeyData = cluster.Config[commonBean.TlsKey] - clusterConfig.CertData = cluster.Config[commonBean.CertData] - clusterConfig.CaData = cluster.Config[commonBean.CertificateAuthorityData] + sanitizedK8sVersion, err := impl.getSanitizedK8sVersion(referenceTemplate) + if err != nil { + return false, nil, err } - releaseIdentifier := &gRPC.ReleaseIdentifier{ - ReleaseName: releaseName, - ReleaseNamespace: envOverride.Namespace, - ClusterConfig: clusterConfig, + referenceChartByte, err = impl.getReferenceChartByteForHelmTypeApp(envOverride, chartMetaData, referenceTemplatePath, overrideRequest, valuesOverrideResponse) + if err != nil { + impl.logger.Errorw("error, getReferenceChartByteForHelmTypeApp", "envOverride", envOverride, "err", err) + return false, nil, err } - if pipelineModel.DeploymentAppCreated { req := &gRPC.UpgradeReleaseRequest{ ReleaseIdentifier: releaseIdentifier, ValuesYaml: mergeAndSave, - HistoryMax: impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_DEVTRON_APP), + HistoryMax: helmRevisionHistory, ChartContent: &gRPC.ChartContent{Content: referenceChartByte}, } if len(sanitizedK8sVersion) > 0 { @@ -1188,7 +1166,7 @@ func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, } //update workflow runner status, used in app workflow view - err := impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) + err = impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) if err != nil { impl.logger.Errorw("error in updating the workflow runner status, createHelmAppForCdPipeline", "err", err) return false, nil, err @@ -1255,7 +1233,7 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline impl.logger.Errorw("unable to get ArgoCd app", "app", argoAppName, "pipeline", pipeline.Name, "err", err) return false, err } - //if status, ok:=status.FromError(err);ok{ + // if status, ok:=status.FromError(err);ok{ appStatus, _ := status2.FromError(err) if appStatus.Code() == codes.OK { impl.logger.Debugw("argo app exists", "app", argoAppName, "pipeline", pipeline.Name) @@ -1316,7 +1294,7 @@ func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(ctx context.Cont if pipeline.DeploymentAppCreated { return argoAppName, nil } else { - //create + // create appNamespace := envConfigOverride.Namespace if appNamespace == "" { appNamespace = "default" @@ -1342,7 +1320,7 @@ func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(ctx context.Cont if err != nil { return "", err } - //update cd pipeline to mark deployment app created + // update cd pipeline to mark deployment app created _, err = impl.updatePipeline(pipeline, userId) if err != nil { impl.logger.Errorw("error in update cd pipeline for deployment app created or not", "err", err) @@ -1381,10 +1359,6 @@ func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Co return impl.helmAppClient.InstallReleaseWithCustomChart(newCtx, &helmInstallRequest) } -func (impl *TriggerServiceImpl) getEnrichedWorkflowRunner(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, wfrId int) *pipelineConfig.CdWorkflowRunner { - return nil -} - func (impl *TriggerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId, wfrId int) { event, err := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) @@ -1400,12 +1374,13 @@ func (impl *TriggerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.Value } deploymentEvent := app.DeploymentEvent{ ApplicationId: overrideRequest.AppId, - EnvironmentId: overrideRequest.EnvId, //check for production Environment + EnvironmentId: overrideRequest.EnvId, // check for production Environment ReleaseId: releaseId, PipelineOverrideId: pipelineOverrideId, TriggerTime: time.Now(), CiArtifactId: overrideRequest.CiArtifactId, } + ciPipelineMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(artifact.PipelineId) if err != nil { impl.logger.Errorw("error in ") @@ -1456,7 +1431,7 @@ func (impl *TriggerServiceImpl) markImageScanDeployed(ctx context.Context, appId ot, err := impl.imageScanDeployInfoReadService.FetchByAppIdAndEnvId(appId, envId, []string{repository6.ScanObjectType_APP}) if err == pg.ErrNoRows && !isScanEnabled { - //ignoring if no rows are found and scan is disabled + // ignoring if no rows are found and scan is disabled return nil } @@ -1501,17 +1476,14 @@ func (impl *TriggerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync b return impl.globalEnvVariables.EnableAsyncHelmInstallDevtronChart && !forceSync } -func (impl *TriggerServiceImpl) isDevtronAsyncArgoCdInstallModeEnabled(forceSync bool) bool { - return impl.globalEnvVariables.EnableAsyncArgoCdInstallDevtronChart && !forceSync -} - -func (impl *TriggerServiceImpl) isDevtronAsyncInstallModeEnabled(deploymentAppType string, forceSync bool) bool { - if util.IsHelmApp(deploymentAppType) { - return impl.isDevtronAsyncHelmInstallModeEnabled(forceSync) - } else if util.IsAcdApp(deploymentAppType) { - return impl.isDevtronAsyncArgoCdInstallModeEnabled(forceSync) +func (impl *TriggerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { + if util.IsHelmApp(overrideRequest.DeploymentAppType) { + return impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment), nil + } else if util.IsAcdApp(overrideRequest.DeploymentAppType) { + return impl.isDevtronAsyncArgoCdInstallModeEnabledForApp(overrideRequest.AppId, + overrideRequest.EnvId, overrideRequest.ForceSyncDeployment) } - return false + return false, nil } func (impl *TriggerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { @@ -1562,3 +1534,51 @@ func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelin } return nil } + +func (impl *TriggerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { + var sanitizedK8sVersion string + //handle specific case for all cronjob charts from cronjob-chart_1-2-0 to cronjob-chart_1-5-0 where semverCompare + //comparison func has wrong api version mentioned, so for already installed charts via these charts that comparison + //is always false, handles the gh issue:- https://github.com/devtron-labs/devtron/issues/4860 + cronJobChartRegex := regexp.MustCompile(bean.CronJobChartRegexExpression) + if cronJobChartRegex.MatchString(referenceTemplate) { + k8sServerVersion, err := impl.K8sUtil.GetKubeVersion() + if err != nil { + impl.logger.Errorw("exception caught in getting k8sServerVersion", "err", err) + return "", err + } + sanitizedK8sVersion = k8s2.StripPrereleaseFromK8sVersion(k8sServerVersion.String()) + } + return sanitizedK8sVersion, nil +} + +func (impl *TriggerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, + chartMetaData *chart.Metadata, referenceTemplatePath string, overrideRequest *bean3.ValuesOverrideRequest, + valuesOverrideResponse *app.ValuesOverrideResponse) ([]byte, error) { + referenceChartByte := envOverride.Chart.ReferenceChart + // here updating reference chart into database. + if len(envOverride.Chart.ReferenceChart) == 0 { + refChartByte, err := impl.chartTemplateService.GetByteArrayRefChart(chartMetaData, referenceTemplatePath) + if err != nil { + impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) + return nil, err + } + ch := envOverride.Chart + ch.ReferenceChart = refChartByte + ch.UpdatedOn = time.Now() + ch.UpdatedBy = overrideRequest.UserId + err = impl.chartRepository.Update(ch) + if err != nil { + impl.logger.Errorw("chart update error", "err", err, "req", overrideRequest) + return nil, err + } + referenceChartByte = refChartByte + } + var err error + referenceChartByte, err = impl.overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse, chartMetaData, referenceTemplatePath, referenceChartByte) + if err != nil { + impl.logger.Errorw("ref chart commit error on cd trigger", "err", err, "req", overrideRequest) + return nil, err + } + return referenceChartByte, nil +} diff --git a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go new file mode 100644 index 0000000000..a548e15c36 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go @@ -0,0 +1,107 @@ +package devtronApps + +import ( + "context" + "github.com/devtron-labs/common-lib/utils/k8s/commonBean" + bean3 "github.com/devtron-labs/devtron/api/bean" + bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/app" + bean4 "github.com/devtron-labs/devtron/pkg/app/bean" + bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "helm.sh/helm/v3/pkg/chart" +) + +func (impl *TriggerServiceImpl) getEnrichedWorkflowRunner(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, wfrId int) *pipelineConfig.CdWorkflowRunner { + return nil +} + +func (impl *TriggerServiceImpl) postDeployHook(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, referenceChartByte []byte, err error) { + impl.logger.Debugw("no post deploy hook registered") +} + +func (impl *TriggerServiceImpl) isDevtronAsyncArgoCdInstallModeEnabledForApp(appId, envId int, forceSync bool) (bool, error) { + return impl.globalEnvVariables.EnableAsyncArgoCdInstallDevtronChart && !forceSync, nil +} + +func (impl *TriggerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, + valuesOverrideResponse *app.ValuesOverrideResponse) (*chart.Metadata, int32, *gRPC.ReleaseIdentifier, error) { + pipelineModel := valuesOverrideResponse.Pipeline + envOverride := valuesOverrideResponse.EnvOverride + + releaseName := pipelineModel.DeploymentAppName + //getting cluster by id + cluster, err := impl.clusterRepository.FindById(envOverride.Environment.ClusterId) + if err != nil { + impl.logger.Errorw("error in getting cluster by id", "clusterId", envOverride.Environment.ClusterId, "err", err) + return nil, 0, nil, err + } else if cluster == nil { + impl.logger.Errorw("error in getting cluster by id, found nil object", "clusterId", envOverride.Environment.ClusterId) + return nil, 0, nil, err + } + bearerToken := cluster.Config[commonBean.BearerToken] + clusterConfig := &gRPC.ClusterConfig{ + ClusterName: cluster.ClusterName, + Token: bearerToken, + ApiServerUrl: cluster.ServerUrl, + InsecureSkipTLSVerify: cluster.InsecureSkipTlsVerify, + } + if cluster.InsecureSkipTlsVerify == false { + clusterConfig.KeyData = cluster.Config[commonBean.TlsKey] + clusterConfig.CertData = cluster.Config[commonBean.CertData] + clusterConfig.CaData = cluster.Config[commonBean.CertificateAuthorityData] + } + releaseIdentifier := &gRPC.ReleaseIdentifier{ + ReleaseName: releaseName, + ReleaseNamespace: envOverride.Namespace, + ClusterConfig: clusterConfig, + } + + chartMetadata := &chart.Metadata{ + Name: pipelineModel.App.AppName, + Version: envOverride.Chart.ChartVersion, + } + + return chartMetadata, impl.helmAppService.GetRevisionHistoryMaxValue(bean6.SOURCE_DEVTRON_APP), releaseIdentifier, nil +} + +func (impl *TriggerServiceImpl) overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse *app.ValuesOverrideResponse, + chartMetaData *chart.Metadata, referenceTemplatePath string, referenceChartByte []byte) ([]byte, error) { + return referenceChartByte, nil +} + +func (impl *TriggerServiceImpl) getManifestPushService(storageType string) publish.ManifestPushService { + var manifestPushService publish.ManifestPushService + if storageType == bean2.ManifestStorageGit { + manifestPushService = impl.gitOpsManifestPushService + } + return manifestPushService +} + +func (impl *TriggerServiceImpl) preStageHandlingForTriggerStageInBulk(triggerRequest *bean.TriggerRequest) error { + return nil +} + +func (impl *TriggerServiceImpl) manifestGenerationFailedTimelineHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, err error) { +} + +func (impl *TriggerServiceImpl) getHelmManifestForTriggerRelease(ctx context.Context, triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, + valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) ([]byte, error) { + return nil, nil +} + +func (impl *TriggerServiceImpl) buildManifestPushTemplateForNonGitStorageType(overrideRequest *bean3.ValuesOverrideRequest, + valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, err error, manifestPushConfig *repository.ManifestPushConfig, + manifestPushTemplate *bean4.ManifestPushTemplate) error { + return nil +} + +func (impl *TriggerServiceImpl) triggerReleaseSuccessHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, + valuesOverrideResponse *app.ValuesOverrideResponse, helmManifest []byte) error { + return nil +} diff --git a/pkg/eventProcessor/in/CDPipelineEventProcessorService.go b/pkg/eventProcessor/in/CDPipelineEventProcessorService.go index 29596ec22d..d1b6add0aa 100644 --- a/pkg/eventProcessor/in/CDPipelineEventProcessorService.go +++ b/pkg/eventProcessor/in/CDPipelineEventProcessorService.go @@ -86,7 +86,7 @@ func (impl *CDPipelineEventProcessorImpl) SubscribeCDBulkTriggerTopic() error { ReferenceId: pointer.String(msg.MsgId), Context: ctx, } - _, _, err = impl.cdTriggerService.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) + _, _, _, err = impl.cdTriggerService.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) if err != nil { impl.logger.Errorw("Error triggering CD", "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, "msg", msg.Data, "err", err) } diff --git a/pkg/eventProcessor/out/WorkflowEventPublishService.go b/pkg/eventProcessor/out/WorkflowEventPublishService.go index 32a3c7d5c3..4f7d228616 100644 --- a/pkg/eventProcessor/out/WorkflowEventPublishService.go +++ b/pkg/eventProcessor/out/WorkflowEventPublishService.go @@ -26,6 +26,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" internalUtil "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" + appBean "github.com/devtron-labs/devtron/pkg/app/bean" "github.com/devtron-labs/devtron/pkg/app/status" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/eventProcessor/celEvaluator" @@ -40,8 +41,7 @@ import ( type WorkflowEventPublishService interface { TriggerBulkHibernateAsync(request bean.StopDeploymentGroupRequest) (interface{}, error) - TriggerAsyncRelease(userDeploymentRequestId int, overrideRequest *apiBean.ValuesOverrideRequest, - valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredBy int32) (releaseNo int, err error) + TriggerAsyncRelease(userDeploymentRequestId int, overrideRequest *apiBean.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredBy int32) (releaseNo int, manifestPushTemplate *appBean.ManifestPushTemplate, err error) TriggerBulkDeploymentAsync(requests []*bean.BulkTriggerRequest, UserId int32) (interface{}, error) } @@ -117,14 +117,13 @@ func (impl *WorkflowEventPublishServiceImpl) TriggerBulkHibernateAsync(request b } // TriggerAsyncRelease will publish async Install/Upgrade request event for Devtron App releases -func (impl *WorkflowEventPublishServiceImpl) TriggerAsyncRelease(userDeploymentRequestId int, overrideRequest *apiBean.ValuesOverrideRequest, - valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredBy int32) (releaseNo int, err error) { +func (impl *WorkflowEventPublishServiceImpl) TriggerAsyncRelease(userDeploymentRequestId int, overrideRequest *apiBean.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredBy int32) (releaseNo int, manifestPushTemplate *appBean.ManifestPushTemplate, err error) { newCtx, span := otel.Tracer("orchestrator").Start(ctx, "WorkflowEventPublishServiceImpl.TriggerAsyncRelease") defer span.End() topic, msg, err := impl.getAsyncDeploymentTopicAndPayload(userDeploymentRequestId, overrideRequest.DeploymentAppType, valuesOverrideResponse) if err != nil { impl.logger.Errorw("error in fetching values for trigger", "err", err) - return releaseNo, err + return releaseNo, manifestPushTemplate, err } // publish nats event for async installation err = impl.pubSubClient.Publish(topic, msg) @@ -135,21 +134,21 @@ func (impl *WorkflowEventPublishServiceImpl) TriggerAsyncRelease(userDeploymentR if err1 != nil { impl.logger.Errorw("error in updating the workflow runner status, TriggerAsyncRelease", "err", err1) } - return 0, err + return 0, manifestPushTemplate, err } //update workflow runner status, used in app workflow view err = impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInQueue) if err != nil { impl.logger.Errorw("error in updating the workflow runner status, TriggerAsyncRelease", "err", err) - return 0, err + return 0, manifestPushTemplate, err } err = impl.cdWorkflowCommonService.UpdatePreviousQueuedRunnerStatus(overrideRequest.WfrId, overrideRequest.PipelineId, triggeredBy) if err != nil { impl.logger.Errorw("error in updating the previous queued workflow runner status, TriggerAsyncRelease", "err", err) - return 0, err + return 0, manifestPushTemplate, err } - return 0, nil + return 0, manifestPushTemplate, nil } func (impl *WorkflowEventPublishServiceImpl) TriggerBulkDeploymentAsync(requests []*bean.BulkTriggerRequest, UserId int32) (interface{}, error) { diff --git a/pkg/pipeline/AppArtifactManager.go b/pkg/pipeline/AppArtifactManager.go index 7cabb6dbe8..727d734f91 100644 --- a/pkg/pipeline/AppArtifactManager.go +++ b/pkg/pipeline/AppArtifactManager.go @@ -434,7 +434,7 @@ func (impl *AppArtifactManagerImpl) extractParentMetaDataByPipeline(pipeline *pi } if stage == bean.CD_WORKFLOW_TYPE_DEPLOY { - pipelinePreStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(pipeline.Id, repository2.PIPELINE_STAGE_TYPE_PRE_CD) + pipelinePreStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(pipeline.Id, repository2.PIPELINE_STAGE_TYPE_PRE_CD, false) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching PRE-CD stage by cd pipeline id", "pipelineId", pipeline.Id, "err", err) return parentId, parentType, parentCdId, err diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index fc8b729136..68e4589646 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -108,6 +108,7 @@ type CiCdPipelineOrchestrator interface { AddPipelineToTemplate(createRequest *bean.CiConfigRequest, isSwitchCiPipelineRequest bool) (resp *bean.CiConfigRequest, err error) GetSourceCiDownStreamFilters(ctx context.Context, sourceCiPipelineId int) (*bean2.SourceCiDownStreamEnv, error) GetSourceCiDownStreamInfo(ctx context.Context, sourceCIPipeline int, req *bean2.SourceCiDownStreamFilters) (pagination.PaginatedResponse[bean2.SourceCiDownStreamResponse], error) + GetSourceCiPipelineForArtifact(ciPipeline pipelineConfig.CiPipeline) (*pipelineConfig.CiPipeline, error) GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensor.WebhookAndCiData, error) } @@ -2344,6 +2345,23 @@ func (impl CiCdPipelineOrchestratorImpl) GetSourceCiDownStreamInfo(ctx context.C return response, nil } +func (impl *CiCdPipelineOrchestratorImpl) GetSourceCiPipelineForArtifact(ciPipeline pipelineConfig.CiPipeline) (*pipelineConfig.CiPipeline, error) { + sourceCiPipeline := &ciPipeline + if adapter.IsLinkedCD(ciPipeline) { + sourceCdPipeline, err := impl.pipelineRepository.FindById(ciPipeline.ParentCiPipeline) + if err != nil { + impl.logger.Errorw("error in finding source cdPipeline for linked cd", "cdPipelineId", ciPipeline.ParentCiPipeline, "err", err) + return nil, err + } + sourceCiPipeline, err = impl.ciPipelineRepository.FindOneWithAppData(sourceCdPipeline.CiPipelineId) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error in finding ciPipeline for the cd pipeline", "CiPipelineId", sourceCdPipeline.Id, "CiPipelineId", sourceCdPipeline.CiPipelineId, "err", err) + return nil, err + } + } + return sourceCiPipeline, nil +} + func (impl *CiCdPipelineOrchestratorImpl) GetGitCommitEnvVarDataForCICDStage(gitTriggers map[int]pipelineConfig.GitCommit) (map[string]string, *gitSensor.WebhookAndCiData, error) { var webhookAndCiData *gitSensor.WebhookAndCiData var err error diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 07b39de734..6966870101 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -519,7 +519,7 @@ func (impl *CiServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, } func (impl *CiServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { - _, err := impl.workflowService.SubmitWorkflow(workflowRequest) + _, _, err := impl.workflowService.SubmitWorkflow(workflowRequest) if err != nil { impl.Logger.Errorw("workflow error", "err", err) return err diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 356acde271..61493151c8 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -1407,7 +1407,7 @@ func (impl *CdPipelineConfigServiceImpl) RetrieveParentDetails(pipelineId int) ( if workflow.ParentType == appWorkflow.CDPIPELINE { // workflow is of type CD, check for stage // for older apps post cd script was stored in post_stage_config_yaml, for newer apps new stage is created in pipeline_stage - parentPostStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(workflow.ParentId, repository5.PIPELINE_STAGE_TYPE_POST_CD) + parentPostStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(workflow.ParentId, repository5.PIPELINE_STAGE_TYPE_POST_CD, false) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching post stage by pipeline id", "err", err, "cd-pipeline-id", parentId) return workflow.ParentId, bean2.CD_WORKFLOW_TYPE_DEPLOY, err diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index e7c58bddbc..2ff4123f0f 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -48,7 +48,7 @@ type PipelineStageService interface { GetCiPipelineStageDataDeepCopy(ciPipelineId int) (preCiStage *bean.PipelineStageDto, postCiStage *bean.PipelineStageDto, err error) GetCdPipelineStageDataDeepCopy(cdPipeline *pipelineConfig.Pipeline) (*bean.PipelineStageDto, *bean.PipelineStageDto, error) GetCdPipelineStageDataDeepCopyForPipelineIds(cdPipelineIds []int, pipelineMap map[int]*pipelineConfig.Pipeline) (map[int][]*bean.PipelineStageDto, error) - GetCdStageByCdPipelineIdAndStageType(cdPipelineId int, stageType repository.PipelineStageType) (*repository.PipelineStage, error) + GetCdStageByCdPipelineIdAndStageType(cdPipelineId int, stageType repository.PipelineStageType, enforcePluginPolicyV2 bool) (*repository.PipelineStage, error) // DeletePipelineStageIfReq function is used to delete corrupted pipelineStage data // , there was a bug(https://github.com/devtron-labs/devtron/issues/3826) where we were not deleting pipeline stage entry even after deleting all the pipelineStageSteps // , this will delete those pipelineStage entry @@ -113,7 +113,7 @@ func (impl *PipelineStageServiceImpl) GetCiPipelineStageDataDeepCopy(ciPipelineI return preCiStage, postCiStage, nil } -func (impl *PipelineStageServiceImpl) GetCdStageByCdPipelineIdAndStageType(cdPipelineId int, stageType repository.PipelineStageType) (*repository.PipelineStage, error) { +func (impl *PipelineStageServiceImpl) GetCdStageByCdPipelineIdAndStageType(cdPipelineId int, stageType repository.PipelineStageType, enforcePluginPolicyV2 bool) (*repository.PipelineStage, error) { return impl.pipelineStageRepository.GetCdStageByCdPipelineIdAndStageType(cdPipelineId, stageType) } diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index cb560beb7c..3aa4f83efa 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -49,7 +49,7 @@ import ( // TODO: move isCi/isJob to workflowRequest type WorkflowService interface { - SubmitWorkflow(workflowRequest *types.WorkflowRequest) (*unstructured.UnstructuredList, error) + SubmitWorkflow(workflowRequest *types.WorkflowRequest) (*unstructured.UnstructuredList, string, error) // DeleteWorkflow(wfName string, namespace string) error GetWorkflow(executorType cdWorkflow.WorkflowExecutorType, name string, namespace string, restConfig *rest.Config) (*unstructured.UnstructuredList, error) GetWorkflowStatus(executorType cdWorkflow.WorkflowExecutorType, name string, namespace string, restConfig *rest.Config) (*types.WorkflowStatus, error) @@ -108,17 +108,18 @@ const ( postCdStage = "postCD" ) -func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *types.WorkflowRequest) (*unstructured.UnstructuredList, error) { +func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *types.WorkflowRequest) (*unstructured.UnstructuredList, string, error) { workflowTemplate, err := impl.createWorkflowTemplate(workflowRequest) if err != nil { - return nil, err + return nil, "", err } workflowExecutor := impl.getWorkflowExecutor(workflowRequest.WorkflowExecutor) if workflowExecutor == nil { - return nil, errors.New("workflow executor not found") + return nil, "", errors.New("workflow executor not found") } createdWf, err := workflowExecutor.ExecuteWorkflow(workflowTemplate) - return createdWf, err + jobHelmPackagePath := "" // due to ENT + return createdWf, jobHelmPackagePath, err } func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.WorkflowRequest) (bean3.WorkflowTemplate, error) { diff --git a/pkg/pipeline/WorkflowServiceIT_test.go b/pkg/pipeline/WorkflowServiceIT_test.go index 38feeecfda..c025abe1fe 100644 --- a/pkg/pipeline/WorkflowServiceIT_test.go +++ b/pkg/pipeline/WorkflowServiceIT_test.go @@ -209,7 +209,7 @@ func TestWorkflowServiceImpl_SubmitWorkflow(t *testing.T) { WorkflowExecutor: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_AWF, } - data, _ := workflowServiceImpl.SubmitWorkflow(&workflowRequest) + data, _, _ := workflowServiceImpl.SubmitWorkflow(&workflowRequest) createdWf := &v1alpha1.Workflow{} obj := data.Items[0].Object @@ -345,7 +345,7 @@ func TestWorkflowServiceImpl_SubmitWorkflow(t *testing.T) { WorkflowExecutor: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_AWF, } - data, _ := workflowServiceImpl.SubmitWorkflow(&workflowRequest) + data, _, _ := workflowServiceImpl.SubmitWorkflow(&workflowRequest) createdWf := &v1alpha1.Workflow{} obj := data.Items[0].Object @@ -513,7 +513,7 @@ func TestWorkflowServiceImpl_SubmitWorkflow(t *testing.T) { WorkflowExecutor: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_AWF, } - data, _ := workflowServiceImpl.SubmitWorkflow(&workflowRequest) + data, _, _ := workflowServiceImpl.SubmitWorkflow(&workflowRequest) createdWf := &v1alpha1.Workflow{} obj := data.Items[0].Object @@ -742,7 +742,7 @@ func TestWorkflowServiceImpl_SubmitWorkflow(t *testing.T) { WorkflowExecutor: cdWorkflow.WORKFLOW_EXECUTOR_TYPE_AWF, } - data, _ := workflowServiceImpl.SubmitWorkflow(&workflowRequest) + data, _, _ := workflowServiceImpl.SubmitWorkflow(&workflowRequest) createdWf := &v1alpha1.Workflow{} obj := data.Items[0].Object diff --git a/pkg/pipeline/adapter/adapter.go b/pkg/pipeline/adapter/adapter.go index 12b049b3e5..f709c84c20 100644 --- a/pkg/pipeline/adapter/adapter.go +++ b/pkg/pipeline/adapter/adapter.go @@ -192,13 +192,19 @@ func IsLinkedCD(ci pipelineConfig.CiPipeline) bool { } // IsLinkedCI will return if the pipelineConfig.CiPipeline is a Linked CI -func IsLinkedCI(ci pipelineConfig.CiPipeline) bool { +func IsLinkedCI(ci *pipelineConfig.CiPipeline) bool { + if ci == nil { + return false + } return ci.ParentCiPipeline != 0 && ci.PipelineType == string(bean2.LINKED) } // IsCIJob will return if the pipelineConfig.CiPipeline is a CI JOB -func IsCIJob(ci pipelineConfig.CiPipeline) bool { +func IsCIJob(ci *pipelineConfig.CiPipeline) bool { + if ci == nil { + return false + } return ci.PipelineType == string(bean2.CI_JOB) } diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 40a8584394..c25c087cfa 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -243,13 +243,13 @@ func (impl *WorkflowDagExecutorImpl) HandleCdStageReTrigger(runner *pipelineConf } if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - err = impl.cdTriggerService.TriggerPreStage(triggerRequest) + _, err = impl.cdTriggerService.TriggerPreStage(triggerRequest) if err != nil { impl.logger.Errorw("error in TriggerPreStage ", "err", err, "cdWorkflowRunnerId", runner.Id) return err } } else if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { - err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + _, err = impl.cdTriggerService.TriggerPostStage(triggerRequest) if err != nil { impl.logger.Errorw("error in TriggerPostStage ", "err", err, "cdWorkflowRunnerId", runner.Id) return err @@ -402,7 +402,7 @@ func (impl *WorkflowDagExecutorImpl) ProcessDevtronAsyncInstallRequest(cdAsyncIn impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", err) return err } - releaseId, releaseErr := impl.cdTriggerService.TriggerRelease(overrideRequest, envDeploymentConfig, newCtx, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) + releaseId, _, releaseErr := impl.cdTriggerService.TriggerRelease(overrideRequest, envDeploymentConfig, newCtx, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) if releaseErr != nil { impl.logger.Errorw("error encountered in ProcessDevtronAsyncInstallRequest", "err", releaseErr, "cdWfrId", cdWfr.Id) impl.handleAsyncTriggerReleaseError(newCtx, releaseErr, cdWfr, overrideRequest) @@ -530,7 +530,7 @@ func (impl *WorkflowDagExecutorImpl) triggerIfAutoStageCdPipeline(request trigge // pre stage exists if request.Pipeline.PreTriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) - err = impl.cdTriggerService.TriggerPreStage(request) // TODO handle error here + _, err = impl.cdTriggerService.TriggerPreStage(request) // TODO handle error here return err } } else if request.Pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { @@ -543,7 +543,7 @@ func (impl *WorkflowDagExecutorImpl) triggerIfAutoStageCdPipeline(request trigge } func (impl *WorkflowDagExecutorImpl) getPipelineStage(pipelineId int, stageType repository4.PipelineStageType) (*repository4.PipelineStage, error) { - stage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(pipelineId, stageType) + stage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(pipelineId, stageType, false) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching CD pipeline stage", "cdPipelineId", pipelineId, "stage ", stage, "err", err) return nil, err @@ -606,7 +606,7 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext t } else { ciArtifactId = cdStageCompleteEvent.CiArtifactDTO.Id } - err = impl.cdTriggerService.TriggerAutoCDOnPreStageSuccess(triggerContext, cdStageCompleteEvent.CdPipelineId, ciArtifactId, cdStageCompleteEvent.WorkflowId, cdStageCompleteEvent.TriggeredBy, 0) + err = impl.cdTriggerService.TriggerAutoCDOnPreStageSuccess(triggerContext, cdStageCompleteEvent.CdPipelineId, ciArtifactId, cdStageCompleteEvent.WorkflowId, cdStageCompleteEvent.TriggeredBy) if err != nil { impl.logger.Errorw("error in triggering cd on pre cd succcess", "err", err) return err @@ -651,7 +651,7 @@ func (impl *WorkflowDagExecutorImpl) HandleDeploymentSuccessEvent(triggerContext RefCdWorkflowRunnerId: 0, } triggerRequest.TriggerContext.Context = context.Background() - err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + _, err = impl.cdTriggerService.TriggerPostStage(triggerRequest) if err != nil { impl.logger.Errorw("error in triggering post stage after successful deployment event", "err", err, "cdWorkflow", cdWorkflow) return err diff --git a/wire_gen.go b/wire_gen.go index 874db71a0b..45b0505c63 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -711,7 +711,7 @@ func InitializeApp() (*App, error) { manifestPushConfigRepositoryImpl := repository13.NewManifestPushConfigRepository(sugaredLogger, db) scanToolExecutionHistoryMappingRepositoryImpl := repository8.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl) + triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl) if err != nil { return nil, err }