From 93b624eeff7aa4ce96fafe29bebcdebf6ad2462f Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 7 Aug 2024 12:01:57 +0530 Subject: [PATCH 01/23] wip: new plugin creation api and min plugin api with only shared plugin list --- api/restHandler/GlobalPluginRestHandler.go | 59 +++++ api/router/GlobalPluginRouter.go | 5 +- pkg/plugin/GlobalPluginService.go | 247 ++++++++++++++++-- pkg/plugin/adaptor/adaptor.go | 16 ++ pkg/plugin/bean/bean.go | 91 ++++--- .../repository/GlobalPluginRepository.go | 116 +++++++- pkg/plugin/utils/utils.go | 25 ++ 7 files changed, 491 insertions(+), 68 deletions(-) create mode 100644 pkg/plugin/adaptor/adaptor.go diff --git a/api/restHandler/GlobalPluginRestHandler.go b/api/restHandler/GlobalPluginRestHandler.go index 7fc25ec6b1..b08c38eb4f 100644 --- a/api/restHandler/GlobalPluginRestHandler.go +++ b/api/restHandler/GlobalPluginRestHandler.go @@ -35,6 +35,7 @@ import ( type GlobalPluginRestHandler interface { PatchPlugin(w http.ResponseWriter, r *http.Request) + CreatePlugin(w http.ResponseWriter, r *http.Request) GetAllGlobalVariables(w http.ResponseWriter, r *http.Request) ListAllPlugins(w http.ResponseWriter, r *http.Request) @@ -46,6 +47,7 @@ type GlobalPluginRestHandler interface { GetPluginDetailByIds(w http.ResponseWriter, r *http.Request) GetAllUniqueTags(w http.ResponseWriter, r *http.Request) MigratePluginData(w http.ResponseWriter, r *http.Request) + GetAllPluginMinData(w http.ResponseWriter, r *http.Request) } func NewGlobalPluginRestHandler(logger *zap.SugaredLogger, globalPluginService plugin.GlobalPluginService, @@ -420,3 +422,60 @@ func (handler *GlobalPluginRestHandlerImpl) MigratePluginData(w http.ResponseWri } common.WriteJsonResp(w, nil, nil, http.StatusOK) } + +func (handler *GlobalPluginRestHandlerImpl) CreatePlugin(w http.ResponseWriter, r *http.Request) { + decoder := json.NewDecoder(r.Body) + userId, err := handler.userService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + var pluginDataDto *bean.PluginParentMetadataDto + err = decoder.Decode(&pluginDataDto) + if err != nil { + handler.logger.Errorw("request err, CreatePlugin", "error", err, "payload", pluginDataDto) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + handler.logger.Infow("request payload received for patching plugins", pluginDataDto, "userId", userId) + // RBAC enforcer applying + token := r.Header.Get("token") + if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionUpdate, "*"); !ok { + common.WriteJsonResp(w, errors.New("unauthorized user"), nil, http.StatusForbidden) + return + } + + //RBAC enforcer Ends + pluginVersionId, err := handler.globalPluginService.CreatePluginOrVersions(pluginDataDto, userId) + if err != nil { + handler.logger.Errorw("service error, error in creating plugin", "pluginCreateRequestDto", pluginDataDto, "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, pluginVersionId, http.StatusOK) +} + +func (handler *GlobalPluginRestHandlerImpl) GetAllPluginMinData(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("token") + appId, err := common.ExtractIntQueryParam(w, r, "appId", 0) + if err != nil { + return + } + ok, err := handler.IsUserAuthorized(token, appId) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + if !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + return + } + + pluginDetail, err := handler.globalPluginService.GetAllPluginMinData() + if err != nil { + handler.logger.Errorw("error in getting all unique tags", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, pluginDetail, http.StatusOK) +} diff --git a/api/router/GlobalPluginRouter.go b/api/router/GlobalPluginRouter.go index 06950c3342..0d8154bf82 100644 --- a/api/router/GlobalPluginRouter.go +++ b/api/router/GlobalPluginRouter.go @@ -41,7 +41,8 @@ type GlobalPluginRouterImpl struct { func (impl *GlobalPluginRouterImpl) initGlobalPluginRouter(globalPluginRouter *mux.Router) { globalPluginRouter.Path("/migrate"). HandlerFunc(impl.globalPluginRestHandler.MigratePluginData).Methods("PUT") - + globalPluginRouter.Path("/create"). + HandlerFunc(impl.globalPluginRestHandler.CreatePlugin).Methods("POST") // versioning impact handling to be done for below apis, globalPluginRouter.Path(""). HandlerFunc(impl.globalPluginRestHandler.PatchPlugin).Methods("POST") @@ -68,5 +69,7 @@ func (impl *GlobalPluginRouterImpl) initGlobalPluginRouter(globalPluginRouter *m globalPluginRouter.Path("/list/tags"). HandlerFunc(impl.globalPluginRestHandler.GetAllUniqueTags).Methods("GET") + globalPluginRouter.Path("/list/v2/min"). + HandlerFunc(impl.globalPluginRestHandler.GetAllPluginMinData).Methods("GET") } diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index 55ff942789..7b68386849 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -1,17 +1,5 @@ /* * Copyright (c) 2024. Devtron Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ package plugin @@ -24,6 +12,7 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/auth/user/bean" repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/plugin/adaptor" bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" helper2 "github.com/devtron-labs/devtron/pkg/plugin/helper" "github.com/devtron-labs/devtron/pkg/plugin/repository" @@ -31,6 +20,7 @@ import ( "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" "go.uber.org/zap" + "golang.org/x/mod/semver" "net/http" "strconv" "strings" @@ -73,9 +63,11 @@ type GlobalPluginService interface { GetDetailedPluginInfoByPluginId(pluginId int) (*bean2.PluginMetadataDto, error) GetAllDetailedPluginInfo() ([]*bean2.PluginMetadataDto, error) + CreatePluginOrVersions(pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) ListAllPluginsV2(filter *bean2.PluginsListFilter) (*bean2.PluginsDto, error) GetPluginDetailV2(pluginVersionIds, parentPluginIds []int, fetchAllVersionDetails bool) (*bean2.PluginsDto, error) GetAllUniqueTags() (*bean2.PluginTagsDto, error) + GetAllPluginMinData() ([]*bean2.PluginMinDto, error) MigratePluginData() error } @@ -429,7 +421,7 @@ func (impl *GlobalPluginServiceImpl) validatePluginRequest(pluginReq *bean2.Plug return errors.New("invalid plugin type, should be of the type PRESET or SHARED") } - plugins, err := impl.globalPluginRepository.GetMetaDataForAllPlugins() + plugins, err := impl.globalPluginRepository.GetAllPluginMinData() if err != nil { impl.logger.Errorw("error in getting all plugins", "err", err) return err @@ -768,7 +760,6 @@ func (impl *GlobalPluginServiceImpl) updatePlugin(pluginUpdateReq *bean2.PluginM if len(pluginUpdateReq.Type) == 0 { return nil, errors.New("invalid plugin type, should be of the type PRESET or SHARED") } - dbConnection := impl.globalPluginRepository.GetConnection() tx, err := dbConnection.Begin() if err != nil { @@ -1386,7 +1377,6 @@ func filterPluginStepData(existingPluginStepsInDb []*repository.PluginStep, plug } else { return nil, nil, pluginStepUpdateReq } - return newPluginStepsToCreate, pluginStepsToRemove, pluginStepsToUpdate } @@ -1884,7 +1874,6 @@ func (impl *GlobalPluginServiceImpl) MigratePluginData() error { // MigratePluginDataToParentPluginMetadata migrates pre-existing plugin metadata from plugin_metadata table into plugin_parent_metadata table, // and also populate plugin_parent_metadata_id in plugin_metadata. -// this operation will happen only once when the get all plugin list v2 api is being called, returns error if any func (impl *GlobalPluginServiceImpl) MigratePluginDataToParentPluginMetadata(pluginsMetadata []*repository.PluginMetadata) error { dbConnection := impl.globalPluginRepository.GetConnection() tx, err := dbConnection.Begin() @@ -1948,3 +1937,229 @@ func (impl *GlobalPluginServiceImpl) MigratePluginDataToParentPluginMetadata(plu } return nil } + +func (impl *GlobalPluginServiceImpl) GetAllPluginMinData() ([]*bean2.PluginMinDto, error) { + pluginsParentMinData, err := impl.globalPluginRepository.GetAllPluginMinData() + if err != nil { + impl.logger.Errorw("GetAllPluginMinData, error in getting all plugin parent metadata min data", "err", err) + return nil, err + } + pluginMinList := make([]*bean2.PluginMinDto, 0, len(pluginsParentMinData)) + for _, item := range pluginsParentMinData { + if item.Type == repository.PLUGIN_TYPE_PRESET { + continue + } + pluginMinList = append(pluginMinList, bean2.NewPluginMinDto().WithParentPluginId(item.Id).WithPluginName(item.Name).WithIcon(item.Icon)) + } + return pluginMinList, nil +} + +func (impl *GlobalPluginServiceImpl) validateV2PluginRequest(pluginReq *bean2.PluginParentMetadataDto) error { + if len(pluginReq.Type) == 0 { + return &util.ApiError{ + HttpStatusCode: http.StatusBadRequest, + Code: strconv.Itoa(http.StatusBadRequest), + InternalMessage: bean2.InvalidPluginTypeError, + UserMessage: errors.New(bean2.InvalidPluginTypeError), + } + } + if len(pluginReq.Versions.DetailedPluginVersionData) == 0 || pluginReq.Versions.DetailedPluginVersionData[0] == nil { + return &util.ApiError{ + HttpStatusCode: http.StatusBadRequest, + Code: strconv.Itoa(http.StatusBadRequest), + InternalMessage: bean2.NoStepDataToProceedError, + UserMessage: errors.New(bean2.NoStepDataToProceedError), + } + } + plugins, err := impl.globalPluginRepository.GetAllPluginMinData() + if err != nil { + impl.logger.Errorw("error in getting all plugins", "err", err) + return err + } + for _, plugin := range plugins { + if plugin.Name == pluginReq.Name { + return &util.ApiError{ + HttpStatusCode: http.StatusConflict, + Code: strconv.Itoa(http.StatusConflict), + InternalMessage: bean2.PluginWithSameNameExistError, + UserMessage: errors.New(bean2.PluginWithSameNameExistError), + } + } + if plugin.Identifier == pluginReq.PluginIdentifier { + return &util.ApiError{ + HttpStatusCode: http.StatusConflict, + Code: strconv.Itoa(http.StatusConflict), + InternalMessage: bean2.PluginWithSameIdentifierExistsError, + UserMessage: errors.New(bean2.PluginWithSameIdentifierExistsError), + } + } + } + // semantic versioning validation on plugin's version + pluginVersionDto := pluginReq.Versions.DetailedPluginVersionData[0] + if !semver.IsValid(pluginVersionDto.Version) { + return &util.ApiError{ + HttpStatusCode: http.StatusBadRequest, + Code: strconv.Itoa(http.StatusBadRequest), + InternalMessage: bean2.PluginVersionNotSemanticallyCorrectError, + UserMessage: errors.New(bean2.PluginVersionNotSemanticallyCorrectError), + } + } + //validate icon url and size + err = utils.FetchIconAndCheckSize(pluginReq.Icon, bean2.PluginIconMaxSizeInBytes) + if err != nil { + return &util.ApiError{ + HttpStatusCode: http.StatusBadRequest, + Code: strconv.Itoa(http.StatusBadRequest), + InternalMessage: fmt.Sprintf("%s err:= %s", bean2.PluginIconNotCorrectOrReachableError, err.Error()), + UserMessage: errors.New(fmt.Sprintf("%s err:= %s", bean2.PluginIconNotCorrectOrReachableError, err.Error())), + } + } + return nil +} + +func (impl *GlobalPluginServiceImpl) createNewPlugin(tx *pg.Tx, pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { + pluginParentMetadata, err := impl.globalPluginRepository.SavePluginParentMetadata(tx, adaptor.GetPluginParentMetadataDbObject(pluginDto, userId)) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in saving plugin parent metadata", "pluginDto", pluginDto, "err", err) + return 0, err + } + pluginDto.Id = pluginParentMetadata.Id + pluginVersionDto := adaptor.GetPluginVersionMetadataDbObject(pluginDto, userId). + WithPluginParentMetadataId(pluginParentMetadata.Id). + WithIsLatestFlag(true) + + pluginVersionMetadata, err := impl.globalPluginRepository.SavePluginMetadata(pluginVersionDto, tx) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in saving plugin version metadata", "pluginDto", pluginDto, "err", err) + return 0, err + } + pluginDto.Versions.DetailedPluginVersionData[0].Id = pluginVersionMetadata.Id + + pluginStageMapping := &repository.PluginStageMapping{ + PluginId: pluginParentMetadata.Id, + StageType: repository.CI_CD, + AuditLog: sql.NewDefaultAuditLog(userId), + } + _, err = impl.globalPluginRepository.SavePluginStageMapping(pluginStageMapping, tx) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in saving plugin stage mapping", "pluginDto", pluginDto, "err", err) + return 0, err + } + + pluginStepsToCreate := pluginDto.Versions.DetailedPluginVersionData[0].PluginSteps + if len(pluginStepsToCreate) > 0 { + err = impl.saveDeepPluginStepData(pluginVersionMetadata.Id, pluginStepsToCreate, userId, tx) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in saving plugin step data", "err", err) + return 0, err + } + } else { + return 0, util.NewApiError().WithCode(strconv.Itoa(http.StatusBadRequest)).WithHttpStatusCode(http.StatusBadRequest). + WithInternalMessage(bean2.PluginStepsNotProvidedError).WithUserMessage(errors.New(bean2.PluginStepsNotProvidedError)) + } + newTagsPresentInReq := pluginDto.Versions.DetailedPluginVersionData[0].NewTagsPresent + if newTagsPresentInReq { + err = impl.CreateNewPluginTagsAndRelationsIfRequiredV2(pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in CreateNewPluginTagsAndRelationsIfRequired", "tags", pluginDto.Versions.DetailedPluginVersionData[0].Tags, "err", err) + return 0, err + } + } + + return pluginVersionMetadata.Id, nil +} + +func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx *pg.Tx, pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { + + return 0, nil +} + +func (impl *GlobalPluginServiceImpl) createPluginV2(pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { + err := impl.validateV2PluginRequest(pluginDto) + if err != nil { + return 0, err + } + + dbConnection := impl.globalPluginRepository.GetConnection() + tx, err := dbConnection.Begin() + if err != nil { + return 0, err + } + // Rollback tx on error. + defer tx.Rollback() + var versionMetadataId int + if pluginDto.Id > 0 { + // create new version of existing plugin req. + versionMetadataId, err = impl.createNewPluginVersionOfExistingPlugin(tx, pluginDto, userId) + if err != nil { + impl.logger.Errorw("createPluginV2, error in creating new version of an existing plugin", "existingPluginName", pluginDto.Name, "err", err) + return 0, err + } + } else { + // create new plugin req. + versionMetadataId, err = impl.createNewPlugin(tx, pluginDto, userId) + if err != nil { + impl.logger.Errorw("createPluginV2, error in creating new plugin", "pluginDto", pluginDto, "err", err) + return 0, err + } + } + + err = tx.Commit() + if err != nil { + impl.logger.Errorw("createPluginV2, error in committing db transaction", "err", err) + return 0, err + } + return versionMetadataId, nil +} + +func (impl *GlobalPluginServiceImpl) CreatePluginOrVersions(pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { + //pluginMetadataDto := adaptor.GetPluginMetadataDto(pluginDto) + pluginMetadataId, err := impl.createPluginV2(pluginDto, userId) + if err != nil { + impl.logger.Errorw("error in creating plugin", "pluginDto", pluginDto, "err", err) + return 0, err + } + return pluginMetadataId, nil +} + +func (impl *GlobalPluginServiceImpl) CreateNewPluginTagsAndRelationsIfRequiredV2(pluginReq *bean2.PluginsVersionDetail, userId int32, tx *pg.Tx) error { + allPluginTags, err := impl.globalPluginRepository.GetAllPluginTags() + if err != nil { + impl.logger.Errorw("error in getting all plugin tags", "err", err) + return err + } + existingTagMap := make(map[string]bool, len(allPluginTags)) + for _, tag := range allPluginTags { + existingTagMap[tag.Name] = true + } + //check for new tags, then create new plugin_tag and plugin_ tag_relation entry in db when new tags are present in request + newPluginTagsToCreate := make([]*repository.PluginTag, 0, len(pluginReq.Tags)) + newPluginTagRelationsToCreate := make([]*repository.PluginTagRelation, 0, len(pluginReq.Tags)) + + for _, tagReq := range pluginReq.Tags { + if _, ok := existingTagMap[tagReq]; !ok { + newPluginTagsToCreate = append(newPluginTagsToCreate, repository.NewPluginTag().CreateAuditLog(userId).WithName(tagReq)) + } + } + + if len(newPluginTagsToCreate) > 0 { + err = impl.globalPluginRepository.SavePluginTagInBulk(newPluginTagsToCreate, tx) + if err != nil { + impl.logger.Errorw("error in saving plugin tag", "newPluginTags", newPluginTagsToCreate, "err", err) + return err + } + } + + for _, newTag := range newPluginTagsToCreate { + newPluginTagRelationsToCreate = append(newPluginTagRelationsToCreate, repository.NewPluginTagRelation().CreateAuditLog(userId).WithTagAndPluginId(newTag.Id, pluginReq.Id)) + } + + if len(newPluginTagRelationsToCreate) > 0 { + err = impl.globalPluginRepository.SavePluginTagRelationInBulk(newPluginTagRelationsToCreate, tx) + if err != nil { + impl.logger.Errorw("error in saving plugin tag relation in bulk", "newPluginTagRelationsToCreate", newPluginTagRelationsToCreate, "err", err) + return err + } + } + return nil +} diff --git a/pkg/plugin/adaptor/adaptor.go b/pkg/plugin/adaptor/adaptor.go new file mode 100644 index 0000000000..24c7d1e81d --- /dev/null +++ b/pkg/plugin/adaptor/adaptor.go @@ -0,0 +1,16 @@ +package adaptor + +import ( + bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" + "github.com/devtron-labs/devtron/pkg/plugin/repository" +) + +func GetPluginParentMetadataDbObject(pluginDto *bean2.PluginParentMetadataDto, userId int32) *repository.PluginParentMetadata { + return repository.NewPluginParentMetadata().CreateAuditLog(userId). + WithBasicMetadata(pluginDto.Name, pluginDto.PluginIdentifier, pluginDto.Description, pluginDto.Icon, pluginDto.Type) +} + +func GetPluginVersionMetadataDbObject(pluginDto *bean2.PluginParentMetadataDto, userId int32) *repository.PluginMetadata { + versionDto := pluginDto.Versions.DetailedPluginVersionData[0] + return repository.NewPluginVersionMetadata().CreateAuditLog(userId).WithBasicMetadata(pluginDto.Name, versionDto.Description, versionDto.Version, versionDto.DocLink) +} diff --git a/pkg/plugin/bean/bean.go b/pkg/plugin/bean/bean.go index 9e152272e3..d531b99e9b 100644 --- a/pkg/plugin/bean/bean.go +++ b/pkg/plugin/bean/bean.go @@ -1,17 +1,5 @@ /* * Copyright (c) 2024. Devtron Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ package bean @@ -44,15 +32,42 @@ type PluginListComponentDto struct { //created new struct for backward compatibi } type PluginMetadataDto struct { - Id int `json:"id"` - Name string `json:"name"` - Description string `json:"description"` - Type string `json:"type,omitempty" validate:"oneof=SHARED PRESET"` // SHARED, PRESET etc - Icon string `json:"icon,omitempty"` - Tags []string `json:"tags"` - Action int `json:"action,omitempty"` - PluginStage string `json:"pluginStage,omitempty"` - PluginSteps []*PluginStepsDto `json:"pluginSteps,omitempty"` + Id int `json:"id"` + Name string `json:"name" validate:"required,min=3,max=100,global-entity-name"` + Description string `json:"description" validate:"max=300"` + Type string `json:"type,omitempty" validate:"oneof=SHARED PRESET"` // SHARED, PRESET etc + Icon string `json:"icon,omitempty"` + Tags []string `json:"tags"` + Action int `json:"action,omitempty"` + PluginStage string `json:"pluginStage,omitempty"` + PluginSteps []*PluginStepsDto `json:"pluginSteps,omitempty"` + NewTagsPresent bool `json:"newTagsPresent,omitempty"` + //PluginIdentifier string `json:"pluginIdentifier" validate:"required,min=3,max=100,global-entity-name"` +} + +type PluginMinDto struct { + ParentPluginId int `json:"parentPluginId"` + PluginName string `json:"pluginName"` + Icon string `json:"icon"` +} + +func NewPluginMinDto() *PluginMinDto { + return &PluginMinDto{} +} + +func (r *PluginMinDto) WithParentPluginId(id int) *PluginMinDto { + r.ParentPluginId = id + return r +} + +func (r *PluginMinDto) WithPluginName(name string) *PluginMinDto { + r.PluginName = name + return r +} + +func (r *PluginMinDto) WithIcon(icon string) *PluginMinDto { + r.Icon = icon + return r } type PluginsDto struct { @@ -76,9 +91,9 @@ func (r *PluginsDto) WithTotalCount(count int) *PluginsDto { type PluginParentMetadataDto struct { Id int `json:"id"` - Name string `json:"name"` - PluginIdentifier string `json:"pluginIdentifier"` - Description string `json:"description"` + Name string `json:"name" validate:"required,min=3,max=100,global-entity-name"` + PluginIdentifier string `json:"pluginIdentifier" validate:"required,min=3,max=100,global-entity-name"` + Description string `json:"description" validate:"max=300"` Type string `json:"type,omitempty" validate:"oneof=SHARED PRESET"` Icon string `json:"icon,omitempty"` Versions *PluginVersions `json:"pluginVersions"` @@ -124,17 +139,6 @@ type PluginVersions struct { MinimalPluginVersionData []*PluginsVersionDetail `json:"minimalPluginVersionData"` // contains only few metadata } -type PluginMinDto struct { - PluginName string `json:"pluginName"` - PluginVersions []*PluginVersionsMinDto `json:"pluginVersions"` - Icon string `json:"icon"` -} - -type PluginVersionsMinDto struct { - Id int `json:"id"` - Version string `json:"version"` -} - func NewPluginVersions() *PluginVersions { return &PluginVersions{} } @@ -154,7 +158,7 @@ type PluginsVersionDetail struct { InputVariables []*PluginVariableDto `json:"inputVariables"` OutputVariables []*PluginVariableDto `json:"outputVariables"` DocLink string `json:"docLink"` - Version string `json:"pluginVersion"` + Version string `json:"pluginVersion" validate:"max=50,min=3"` IsLatest bool `json:"isLatest"` UpdatedBy string `json:"updatedBy"` CreatedOn time.Time `json:"-"` @@ -171,6 +175,7 @@ func (r *PluginsVersionDetail) SetMinimalPluginsVersionDetail(pluginVersionMetad r.Description = pluginVersionMetadata.Description r.Version = pluginVersionMetadata.PluginVersion r.IsLatest = pluginVersionMetadata.IsLatest + r.DocLink = pluginVersionMetadata.DocLink return r } @@ -335,10 +340,18 @@ type RegistryCredentials struct { } const ( - NoPluginOrParentIdProvidedErr = "no value for pluginVersionIds and parentPluginIds provided in query param" - NoPluginFoundForThisSearchQueryErr = "unable to find desired plugin for the query filter" + NoPluginOrParentIdProvidedErr = "no value for pluginVersionIds and parentPluginIds provided in query param" + NoPluginFoundForThisSearchQueryErr = "unable to find desired plugin for the query filter" + PluginStepsNotProvidedError = "plugin steps not provided" + PluginWithSameNameExistError = "plugin with the same name exists, please choose another name" + PluginWithSameIdentifierExistsError = "plugin with the same identifier exists, please choose another name" + PluginVersionNotSemanticallyCorrectError = "please provide a plugin version that adheres to Semantic Versioning 2.0.0 to ensure compatibility and proper versioning" + PluginIconNotCorrectOrReachableError = "cannot validate icon, make sure that provided url link is reachable" + InvalidPluginTypeError = "invalid plugin type, should be of the type SHARED" + NoStepDataToProceedError = "no step data provided to save, please provide a plugin step to proceed further" ) const ( - SpecialCharsRegex = ` !"#$%&'()*+,./:;<=>?@[\]^_{|}~` + "`" + SpecialCharsRegex = ` !"#$%&'()*+,./:;<=>?@[\]^_{|}~` + "`" + PluginIconMaxSizeInBytes = 2 * 1024 * 1024 ) diff --git a/pkg/plugin/repository/GlobalPluginRepository.go b/pkg/plugin/repository/GlobalPluginRepository.go index 8b65093523..d1d55c73d8 100644 --- a/pkg/plugin/repository/GlobalPluginRepository.go +++ b/pkg/plugin/repository/GlobalPluginRepository.go @@ -1,17 +1,5 @@ /* * Copyright (c) 2024. Devtron Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ package repository @@ -99,6 +87,16 @@ func (r *PluginParentMetadata) CreateAuditLog(userId int32) *PluginParentMetadat return r } +func (r *PluginParentMetadata) WithBasicMetadata(name, identifier, description, icon, pluginType string) *PluginParentMetadata { + r.Name = name + r.Identifier = identifier + r.Description = description + r.Icon = icon + r.Type = PluginType(pluginType) + r.Deleted = false + return r +} + // SetParentPluginMetadata method signature used only for migration purposes, sets pluginVersionsMetadata into plugin_parent_metadata func (r *PluginParentMetadata) SetParentPluginMetadata(pluginMetadata *PluginMetadata) *PluginParentMetadata { r.Name = pluginMetadata.Name @@ -135,6 +133,38 @@ type PluginMetadata struct { sql.AuditLog } +func NewPluginVersionMetadata() *PluginMetadata { + return &PluginMetadata{} +} + +func (r *PluginMetadata) CreateAuditLog(userId int32) *PluginMetadata { + r.CreatedBy = userId + r.CreatedOn = time.Now() + r.UpdatedBy = userId + r.UpdatedOn = time.Now() + return r +} + +func (r *PluginMetadata) WithBasicMetadata(name, description, pluginVersion, docLink string) *PluginMetadata { + r.Name = name + r.PluginVersion = pluginVersion + r.Description = description + r.DocLink = docLink + r.Deleted = false + r.IsDeprecated = false + return r +} + +func (r *PluginMetadata) WithPluginParentMetadataId(parentId int) *PluginMetadata { + r.PluginParentMetadataId = parentId + return r +} + +func (r *PluginMetadata) WithIsLatestFlag(isLatest bool) *PluginMetadata { + r.IsLatest = isLatest + return r +} + type PluginTag struct { tableName struct{} `sql:"plugin_tag" pg:",discard_unknown_columns"` Id int `sql:"id,pk"` @@ -143,6 +173,23 @@ type PluginTag struct { sql.AuditLog } +func NewPluginTag() *PluginTag { + return &PluginTag{} +} + +func (r *PluginTag) WithName(name string) *PluginTag { + r.Name = name + return r +} + +func (r *PluginTag) CreateAuditLog(userId int32) *PluginTag { + r.CreatedBy = userId + r.CreatedOn = time.Now() + r.UpdatedBy = userId + r.UpdatedOn = time.Now() + return r +} + type PluginTagRelation struct { tableName struct{} `sql:"plugin_tag_relation" pg:",discard_unknown_columns"` Id int `sql:"id,pk"` @@ -151,6 +198,24 @@ type PluginTagRelation struct { sql.AuditLog } +func NewPluginTagRelation() *PluginTagRelation { + return &PluginTagRelation{} +} + +func (r *PluginTagRelation) WithTagAndPluginId(tagId, pluginId int) *PluginTagRelation { + r.TagId = tagId + r.PluginId = pluginId + return r +} + +func (r *PluginTagRelation) CreateAuditLog(userId int32) *PluginTagRelation { + r.CreatedBy = userId + r.CreatedOn = time.Now() + r.UpdatedBy = userId + r.UpdatedOn = time.Now() + return r +} + // Below two tables are used at pipeline-steps level too type PluginPipelineScript struct { @@ -247,6 +312,7 @@ type GlobalPluginRepository interface { GetMetaDataForAllPlugins() ([]*PluginMetadata, error) GetMetaDataForPluginWithStageType(stageType int) ([]*PluginMetadata, error) GetMetaDataByPluginId(pluginId int) (*PluginMetadata, error) + GetMetaDataByPluginIds(pluginIds []int) ([]*PluginMetadata, error) GetAllPluginTags() ([]*PluginTag, error) GetAllPluginTagRelations() ([]*PluginTagRelation, error) GetTagsByPluginId(pluginId int) ([]string, error) @@ -268,6 +334,7 @@ type GlobalPluginRepository interface { GetPluginParentMetadataByIdentifier(pluginIdentifier string) (*PluginParentMetadata, error) GetAllFilteredPluginParentMetadata(searchKey string, tags []string) ([]*PluginParentMetadata, error) GetPluginParentMetadataByIds(ids []int) ([]*PluginParentMetadata, error) + GetAllPluginMinData() ([]*PluginParentMetadata, error) SavePluginMetadata(pluginMetadata *PluginMetadata, tx *pg.Tx) (*PluginMetadata, error) SavePluginStageMapping(pluginStageMapping *PluginStageMapping, tx *pg.Tx) (*PluginStageMapping, error) @@ -385,6 +452,18 @@ func (impl *GlobalPluginRepositoryImpl) GetMetaDataByPluginId(pluginId int) (*Pl return &plugin, nil } +func (impl *GlobalPluginRepositoryImpl) GetMetaDataByPluginIds(pluginIds []int) ([]*PluginMetadata, error) { + var plugins []*PluginMetadata + err := impl.dbConnection.Model(&plugins). + Where("deleted = ?", false). + Where("id in (?)", pg.In(pluginIds)).Select() + if err != nil { + impl.logger.Errorw("err in getting plugins by pluginIds", "pluginIds", pluginIds, "err", err) + return nil, err + } + return plugins, nil +} + func (impl *GlobalPluginRepositoryImpl) GetStepsByPluginIds(pluginIds []int) ([]*PluginStep, error) { var pluginSteps []*PluginStep err := impl.dbConnection.Model(&pluginSteps). @@ -749,3 +828,16 @@ func (impl *GlobalPluginRepositoryImpl) GetPluginParentMetadataByIds(ids []int) } return plugins, nil } + +func (impl *GlobalPluginRepositoryImpl) GetAllPluginMinData() ([]*PluginParentMetadata, error) { + var plugins []*PluginParentMetadata + err := impl.dbConnection.Model(&plugins). + Column("plugin_parent_metadata.id", "plugin_parent_metadata.name", "plugin_parent_metadata.type", "plugin_parent_metadata.icon", "plugin_parent_metadata.identifier"). + Where("deleted = ?", false). + Select() + if err != nil { + impl.logger.Errorw("err in getting all plugin parent metadata min data", "err", err) + return nil, err + } + return plugins, nil +} diff --git a/pkg/plugin/utils/utils.go b/pkg/plugin/utils/utils.go index 6d78a29143..168e694d89 100644 --- a/pkg/plugin/utils/utils.go +++ b/pkg/plugin/utils/utils.go @@ -21,9 +21,11 @@ import ( "fmt" bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" "github.com/devtron-labs/devtron/pkg/plugin/repository" + "net/http" "regexp" "sort" "strings" + "time" ) func GetStageType(stageTypeReq string) (int, error) { @@ -72,3 +74,26 @@ func SortPluginsVersionDetailSliceByCreatedOn(pluginsVersionDetail []*bean2.Plug return false }) } + +func FetchIconAndCheckSize(url string, maxSize int64) error { + client := http.Client{ + Timeout: 5 * time.Second, + } + iconResp, err := client.Get(url) + if err != nil { + return fmt.Errorf("error in fetching icon : %s", err.Error()) + } + if iconResp != nil { + if iconResp.StatusCode >= 200 && iconResp.StatusCode < 300 { + if iconResp.ContentLength > maxSize { + return fmt.Errorf("icon size too large") + } + iconResp.Body.Close() + } else { + return fmt.Errorf("error in fetching icon : %s", iconResp.Status) + } + } else { + return fmt.Errorf("error in fetching icon : empty response") + } + return nil +} From da93046147132b415bba94b5c4f380f00c3a79b2 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 7 Aug 2024 13:29:40 +0530 Subject: [PATCH 02/23] wip: create new plugin version code --- pkg/plugin/GlobalPluginService.go | 97 +++++++++---------- pkg/plugin/adaptor/adaptor.go | 64 ++++++++++++ .../repository/GlobalPluginRepository.go | 25 +++++ 3 files changed, 135 insertions(+), 51 deletions(-) diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index 7b68386849..f183398c46 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -662,33 +662,10 @@ func (impl *GlobalPluginServiceImpl) UpdatePluginPipelineScript(dbPluginPipeline func (impl *GlobalPluginServiceImpl) saveDeepPluginStepData(pluginMetadataId int, pluginStepsReq []*bean2.PluginStepsDto, userId int32, tx *pg.Tx) error { for _, pluginStep := range pluginStepsReq { - pluginStepData := &repository.PluginStep{ - PluginId: pluginMetadataId, - Name: pluginStep.Name, - Description: pluginStep.Description, - Index: pluginStep.Index, - StepType: pluginStep.StepType, - RefPluginId: pluginStep.RefPluginId, - OutputDirectoryPath: pluginStep.OutputDirectoryPath, - DependentOnStep: pluginStep.DependentOnStep, - AuditLog: sql.NewDefaultAuditLog(userId), - } + pluginStepData := adaptor.GetPluginStepDbObject(pluginStep, pluginMetadataId, userId) //get the script saved for this plugin step if pluginStep.PluginPipelineScript != nil { - pluginPipelineScript := &repository.PluginPipelineScript{ - Script: pluginStep.PluginPipelineScript.Script, - StoreScriptAt: pluginStep.PluginPipelineScript.StoreScriptAt, - Type: pluginStep.PluginPipelineScript.Type, - DockerfileExists: pluginStep.PluginPipelineScript.DockerfileExists, - MountPath: pluginStep.PluginPipelineScript.MountPath, - MountCodeToContainer: pluginStep.PluginPipelineScript.MountCodeToContainer, - MountCodeToContainerPath: pluginStep.PluginPipelineScript.MountCodeToContainerPath, - MountDirectoryFromHost: pluginStep.PluginPipelineScript.MountDirectoryFromHost, - ContainerImagePath: pluginStep.PluginPipelineScript.ContainerImagePath, - ImagePullSecretType: pluginStep.PluginPipelineScript.ImagePullSecretType, - ImagePullSecret: pluginStep.PluginPipelineScript.ImagePullSecret, - AuditLog: sql.NewDefaultAuditLog(userId), - } + pluginPipelineScript := adaptor.GetPluginPipelineScriptDbObject(pluginStep.PluginPipelineScript, userId) pluginPipelineScript, err := impl.globalPluginRepository.SavePluginPipelineScript(pluginPipelineScript, tx) if err != nil { impl.logger.Errorw("error in saving plugin pipeline script", "pluginPipelineScript", pluginPipelineScript, "err", err) @@ -711,23 +688,7 @@ func (impl *GlobalPluginServiceImpl) saveDeepPluginStepData(pluginMetadataId int pluginStep.Id = pluginStepData.Id //create entry in plugin_step_variable for _, pluginStepVariable := range pluginStep.PluginStepVariable { - pluginStepVariableData := &repository.PluginStepVariable{ - PluginStepId: pluginStepData.Id, - Name: pluginStepVariable.Name, - Format: pluginStepVariable.Format, - Description: pluginStepVariable.Description, - IsExposed: pluginStepVariable.IsExposed, - AllowEmptyValue: pluginStepVariable.AllowEmptyValue, - DefaultValue: pluginStepVariable.DefaultValue, - Value: pluginStepVariable.Value, - VariableType: pluginStepVariable.VariableType, - ValueType: pluginStepVariable.ValueType, - PreviousStepIndex: pluginStepVariable.PreviousStepIndex, - VariableStepIndex: pluginStepVariable.VariableStepIndex, - VariableStepIndexInPlugin: pluginStepVariable.VariableStepIndexInPlugin, - ReferenceVariableName: pluginStepVariable.ReferenceVariableName, - AuditLog: sql.NewDefaultAuditLog(userId), - } + pluginStepVariableData := adaptor.GetPluginStepVariableDbObject(pluginStepData.Id, pluginStepVariable, userId) pluginStepVariableData, err = impl.globalPluginRepository.SavePluginStepVariables(pluginStepVariableData, tx) if err != nil { impl.logger.Errorw("error in saving plugin step variable", "pluginStepVariableData", pluginStepVariableData, "err", err) @@ -736,14 +697,7 @@ func (impl *GlobalPluginServiceImpl) saveDeepPluginStepData(pluginMetadataId int pluginStepVariable.Id = pluginStepVariableData.Id //create entry in plugin_step_condition for _, pluginStepCondition := range pluginStepVariable.PluginStepCondition { - pluginStepConditionData := &repository.PluginStepCondition{ - PluginStepId: pluginStepData.Id, - ConditionVariableId: pluginStepVariableData.Id, - ConditionType: pluginStepCondition.ConditionType, - ConditionalOperator: pluginStepCondition.ConditionalOperator, - ConditionalValue: pluginStepCondition.ConditionalValue, - AuditLog: sql.NewDefaultAuditLog(userId), - } + pluginStepConditionData := adaptor.GetPluginStepConditionDbObject(pluginStepData.Id, pluginStepVariableData.Id, pluginStepCondition, userId) pluginStepConditionData, err = impl.globalPluginRepository.SavePluginStepConditions(pluginStepConditionData, tx) if err != nil { impl.logger.Errorw("error in saving plugin step condition", "pluginStepConditionData", pluginStepConditionData, "err", err) @@ -2070,8 +2024,49 @@ func (impl *GlobalPluginServiceImpl) createNewPlugin(tx *pg.Tx, pluginDto *bean2 } func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx *pg.Tx, pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { + pluginParentMinData, err := impl.globalPluginRepository.GetPluginParentMinDataById(pluginDto.Id) + if err != nil { + impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in getting plugin parent metadata", "pluginDto", pluginDto, "err", err) + return 0, err + } + // before saving new plugin version marking previous version's isLatest as false. + err = impl.globalPluginRepository.MarkPreviousPluginVersionLatestFalse(pluginDto.Id) + if err != nil { + impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in MarkPreviousPluginVersionLatestFalse", "pluginParentId", pluginDto.Id, "err", err) + return 0, err + } + pluginVersionDto := adaptor.GetPluginVersionMetadataDbObject(pluginDto, userId). + WithPluginParentMetadataId(pluginParentMinData.Id). + WithIsLatestFlag(true) + + pluginVersionMetadata, err := impl.globalPluginRepository.SavePluginMetadata(pluginVersionDto, tx) + if err != nil { + impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in saving plugin version metadata", "pluginDto", pluginDto, "err", err) + return 0, err + } + pluginDto.Versions.DetailedPluginVersionData[0].Id = pluginVersionMetadata.Id - return 0, nil + pluginStepsToCreate := pluginDto.Versions.DetailedPluginVersionData[0].PluginSteps + if len(pluginStepsToCreate) > 0 { + err = impl.saveDeepPluginStepData(pluginVersionMetadata.Id, pluginStepsToCreate, userId, tx) + if err != nil { + impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in saving plugin step data", "err", err) + return 0, err + } + } else { + return 0, util.NewApiError().WithCode(strconv.Itoa(http.StatusBadRequest)).WithHttpStatusCode(http.StatusBadRequest). + WithInternalMessage(bean2.PluginStepsNotProvidedError).WithUserMessage(errors.New(bean2.PluginStepsNotProvidedError)) + } + newTagsPresentInReq := pluginDto.Versions.DetailedPluginVersionData[0].NewTagsPresent + if newTagsPresentInReq { + err = impl.CreateNewPluginTagsAndRelationsIfRequiredV2(pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) + if err != nil { + impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in CreateNewPluginTagsAndRelationsIfRequired", "tags", pluginDto.Versions.DetailedPluginVersionData[0].Tags, "err", err) + return 0, err + } + } + + return pluginVersionMetadata.Id, nil } func (impl *GlobalPluginServiceImpl) createPluginV2(pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { diff --git a/pkg/plugin/adaptor/adaptor.go b/pkg/plugin/adaptor/adaptor.go index 24c7d1e81d..e57de1abeb 100644 --- a/pkg/plugin/adaptor/adaptor.go +++ b/pkg/plugin/adaptor/adaptor.go @@ -3,6 +3,7 @@ package adaptor import ( bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" "github.com/devtron-labs/devtron/pkg/plugin/repository" + "github.com/devtron-labs/devtron/pkg/sql" ) func GetPluginParentMetadataDbObject(pluginDto *bean2.PluginParentMetadataDto, userId int32) *repository.PluginParentMetadata { @@ -14,3 +15,66 @@ func GetPluginVersionMetadataDbObject(pluginDto *bean2.PluginParentMetadataDto, versionDto := pluginDto.Versions.DetailedPluginVersionData[0] return repository.NewPluginVersionMetadata().CreateAuditLog(userId).WithBasicMetadata(pluginDto.Name, versionDto.Description, versionDto.Version, versionDto.DocLink) } + +func GetPluginStepDbObject(pluginStepDto *bean2.PluginStepsDto, pluginVersionMetadata int, userId int32) *repository.PluginStep { + return &repository.PluginStep{ + PluginId: pluginVersionMetadata, + Name: pluginStepDto.Name, + Description: pluginStepDto.Description, + Index: pluginStepDto.Index, + StepType: pluginStepDto.StepType, + RefPluginId: pluginStepDto.RefPluginId, + OutputDirectoryPath: pluginStepDto.OutputDirectoryPath, + DependentOnStep: pluginStepDto.DependentOnStep, + AuditLog: sql.NewDefaultAuditLog(userId), + } +} +func GetPluginPipelineScriptDbObject(pluginPipelineScript *bean2.PluginPipelineScript, userId int32) *repository.PluginPipelineScript { + return &repository.PluginPipelineScript{ + Script: pluginPipelineScript.Script, + StoreScriptAt: pluginPipelineScript.StoreScriptAt, + Type: pluginPipelineScript.Type, + DockerfileExists: pluginPipelineScript.DockerfileExists, + MountPath: pluginPipelineScript.MountPath, + MountCodeToContainer: pluginPipelineScript.MountCodeToContainer, + MountCodeToContainerPath: pluginPipelineScript.MountCodeToContainerPath, + MountDirectoryFromHost: pluginPipelineScript.MountDirectoryFromHost, + ContainerImagePath: pluginPipelineScript.ContainerImagePath, + ImagePullSecretType: pluginPipelineScript.ImagePullSecretType, + ImagePullSecret: pluginPipelineScript.ImagePullSecret, + AuditLog: sql.NewDefaultAuditLog(userId), + } + +} + +func GetPluginStepVariableDbObject(pluginStepId int, pluginVariableDto *bean2.PluginVariableDto, userId int32) *repository.PluginStepVariable { + return &repository.PluginStepVariable{ + PluginStepId: pluginStepId, + Name: pluginVariableDto.Name, + Format: pluginVariableDto.Format, + Description: pluginVariableDto.Description, + IsExposed: pluginVariableDto.IsExposed, + AllowEmptyValue: pluginVariableDto.AllowEmptyValue, + DefaultValue: pluginVariableDto.DefaultValue, + Value: pluginVariableDto.Value, + VariableType: pluginVariableDto.VariableType, + ValueType: pluginVariableDto.ValueType, + PreviousStepIndex: pluginVariableDto.PreviousStepIndex, + VariableStepIndex: pluginVariableDto.VariableStepIndex, + VariableStepIndexInPlugin: pluginVariableDto.VariableStepIndexInPlugin, + ReferenceVariableName: pluginVariableDto.ReferenceVariableName, + AuditLog: sql.NewDefaultAuditLog(userId), + } +} + +func GetPluginStepConditionDbObject(stepDataId, pluginStepVariableId int, pluginStepCondition *bean2.PluginStepCondition, + userId int32) *repository.PluginStepCondition { + return &repository.PluginStepCondition{ + PluginStepId: stepDataId, + ConditionVariableId: pluginStepVariableId, + ConditionType: pluginStepCondition.ConditionType, + ConditionalOperator: pluginStepCondition.ConditionalOperator, + ConditionalValue: pluginStepCondition.ConditionalValue, + AuditLog: sql.NewDefaultAuditLog(userId), + } +} diff --git a/pkg/plugin/repository/GlobalPluginRepository.go b/pkg/plugin/repository/GlobalPluginRepository.go index d1d55c73d8..37d7c941e8 100644 --- a/pkg/plugin/repository/GlobalPluginRepository.go +++ b/pkg/plugin/repository/GlobalPluginRepository.go @@ -335,6 +335,8 @@ type GlobalPluginRepository interface { GetAllFilteredPluginParentMetadata(searchKey string, tags []string) ([]*PluginParentMetadata, error) GetPluginParentMetadataByIds(ids []int) ([]*PluginParentMetadata, error) GetAllPluginMinData() ([]*PluginParentMetadata, error) + GetPluginParentMinDataById(id int) (*PluginParentMetadata, error) + MarkPreviousPluginVersionLatestFalse(pluginParentId int) error SavePluginMetadata(pluginMetadata *PluginMetadata, tx *pg.Tx) (*PluginMetadata, error) SavePluginStageMapping(pluginStageMapping *PluginStageMapping, tx *pg.Tx) (*PluginStageMapping, error) @@ -779,6 +781,16 @@ func (impl *GlobalPluginRepositoryImpl) GetPluginParentMetadataByIdentifier(plug return &pluginParentMetadata, nil } +func (impl *GlobalPluginRepositoryImpl) GetPluginParentMinDataById(id int) (*PluginParentMetadata, error) { + var pluginParentMetadata PluginParentMetadata + err := impl.dbConnection.Model(&pluginParentMetadata).Where("id = ?", id). + Where("deleted = ?", false).Select() + if err != nil { + return nil, err + } + return &pluginParentMetadata, nil +} + func (impl *GlobalPluginRepositoryImpl) SavePluginParentMetadata(tx *pg.Tx, pluginParentMetadata *PluginParentMetadata) (*PluginParentMetadata, error) { err := tx.Insert(pluginParentMetadata) return pluginParentMetadata, err @@ -841,3 +853,16 @@ func (impl *GlobalPluginRepositoryImpl) GetAllPluginMinData() ([]*PluginParentMe } return plugins, nil } + +func (impl *GlobalPluginRepositoryImpl) MarkPreviousPluginVersionLatestFalse(pluginParentId int) error { + var model PluginMetadata + _, err := impl.dbConnection.Model(&model). + Set("is_latest = ?", false). + Where("id = (select id from plugin_metadata where plugin_parent_metadata_id = ? and is_latest =true order by created_on desc limit 1)", pluginParentId). + Update() + if err != nil { + impl.logger.Errorw("error in updating last version isLatest as false for a plugin parent id", "pluginParentId", pluginParentId, "err", err) + return err + } + return nil +} From 9b88045a62e331d2d8f11002796bf745430ab1cd Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 7 Aug 2024 13:36:55 +0530 Subject: [PATCH 03/23] wip:plugin type SHARED by default --- pkg/plugin/adaptor/adaptor.go | 2 +- pkg/plugin/repository/GlobalPluginRepository.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/plugin/adaptor/adaptor.go b/pkg/plugin/adaptor/adaptor.go index e57de1abeb..c26c5abecb 100644 --- a/pkg/plugin/adaptor/adaptor.go +++ b/pkg/plugin/adaptor/adaptor.go @@ -8,7 +8,7 @@ import ( func GetPluginParentMetadataDbObject(pluginDto *bean2.PluginParentMetadataDto, userId int32) *repository.PluginParentMetadata { return repository.NewPluginParentMetadata().CreateAuditLog(userId). - WithBasicMetadata(pluginDto.Name, pluginDto.PluginIdentifier, pluginDto.Description, pluginDto.Icon, pluginDto.Type) + WithBasicMetadata(pluginDto.Name, pluginDto.PluginIdentifier, pluginDto.Description, pluginDto.Icon, repository.PLUGIN_TYPE_SHARED) } func GetPluginVersionMetadataDbObject(pluginDto *bean2.PluginParentMetadataDto, userId int32) *repository.PluginMetadata { diff --git a/pkg/plugin/repository/GlobalPluginRepository.go b/pkg/plugin/repository/GlobalPluginRepository.go index 37d7c941e8..2929616b96 100644 --- a/pkg/plugin/repository/GlobalPluginRepository.go +++ b/pkg/plugin/repository/GlobalPluginRepository.go @@ -87,12 +87,12 @@ func (r *PluginParentMetadata) CreateAuditLog(userId int32) *PluginParentMetadat return r } -func (r *PluginParentMetadata) WithBasicMetadata(name, identifier, description, icon, pluginType string) *PluginParentMetadata { +func (r *PluginParentMetadata) WithBasicMetadata(name, identifier, description, icon string, pluginType PluginType) *PluginParentMetadata { r.Name = name r.Identifier = identifier r.Description = description r.Icon = icon - r.Type = PluginType(pluginType) + r.Type = pluginType r.Deleted = false return r } From 6da2832e5a46343d69e37bd5e4d45db5e25c1d3b Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 7 Aug 2024 16:59:45 +0530 Subject: [PATCH 04/23] wip:find plugin either by identifier or by id while creating a new version of existing plugin --- pkg/plugin/GlobalPluginService.go | 20 ++++++++++++++----- .../repository/GlobalPluginRepository.go | 11 ++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index f183398c46..6c822ec79d 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -2024,13 +2024,23 @@ func (impl *GlobalPluginServiceImpl) createNewPlugin(tx *pg.Tx, pluginDto *bean2 } func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx *pg.Tx, pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { - pluginParentMinData, err := impl.globalPluginRepository.GetPluginParentMinDataById(pluginDto.Id) - if err != nil { - impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in getting plugin parent metadata", "pluginDto", pluginDto, "err", err) - return 0, err + var pluginParentMinData *repository.PluginParentMetadata + var err error + if pluginDto.Id > 0 { + pluginParentMinData, err = impl.globalPluginRepository.GetPluginParentMinDataById(pluginDto.Id) + if err != nil { + impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in getting plugin parent metadata", "pluginDto", pluginDto, "err", err) + return 0, err + } + } else { + pluginParentMinData, err = impl.globalPluginRepository.GetPluginParentMinDataByIdentifier(pluginDto.PluginIdentifier) + if err != nil { + impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in getting plugin parent metadata", "pluginDto", pluginDto, "err", err) + return 0, err + } } // before saving new plugin version marking previous version's isLatest as false. - err = impl.globalPluginRepository.MarkPreviousPluginVersionLatestFalse(pluginDto.Id) + err = impl.globalPluginRepository.MarkPreviousPluginVersionLatestFalse(pluginParentMinData.Id) if err != nil { impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in MarkPreviousPluginVersionLatestFalse", "pluginParentId", pluginDto.Id, "err", err) return 0, err diff --git a/pkg/plugin/repository/GlobalPluginRepository.go b/pkg/plugin/repository/GlobalPluginRepository.go index 2929616b96..fcc8ad310b 100644 --- a/pkg/plugin/repository/GlobalPluginRepository.go +++ b/pkg/plugin/repository/GlobalPluginRepository.go @@ -336,6 +336,7 @@ type GlobalPluginRepository interface { GetPluginParentMetadataByIds(ids []int) ([]*PluginParentMetadata, error) GetAllPluginMinData() ([]*PluginParentMetadata, error) GetPluginParentMinDataById(id int) (*PluginParentMetadata, error) + GetPluginParentMinDataByIdentifier(identifier string) (*PluginParentMetadata, error) MarkPreviousPluginVersionLatestFalse(pluginParentId int) error SavePluginMetadata(pluginMetadata *PluginMetadata, tx *pg.Tx) (*PluginMetadata, error) @@ -791,6 +792,16 @@ func (impl *GlobalPluginRepositoryImpl) GetPluginParentMinDataById(id int) (*Plu return &pluginParentMetadata, nil } +func (impl *GlobalPluginRepositoryImpl) GetPluginParentMinDataByIdentifier(identifier string) (*PluginParentMetadata, error) { + var pluginParentMetadata PluginParentMetadata + err := impl.dbConnection.Model(&pluginParentMetadata).Where("identifier = ?", identifier). + Where("deleted = ?", false).Select() + if err != nil { + return nil, err + } + return &pluginParentMetadata, nil +} + func (impl *GlobalPluginRepositoryImpl) SavePluginParentMetadata(tx *pg.Tx, pluginParentMetadata *PluginParentMetadata) (*PluginParentMetadata, error) { err := tx.Insert(pluginParentMetadata) return pluginParentMetadata, err From 7dc5df60be5e9f532ffbc85e4e4cc636b905faad Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 7 Aug 2024 22:59:53 +0530 Subject: [PATCH 05/23] wip: create new plugin tag logic improved --- pkg/plugin/GlobalPluginService.go | 92 ++++++++----------- pkg/plugin/bean/bean.go | 2 +- .../repository/GlobalPluginRepository.go | 11 --- 3 files changed, 37 insertions(+), 68 deletions(-) diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index 6c822ec79d..719690c6ef 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -801,6 +801,7 @@ func (impl *GlobalPluginServiceImpl) updatePlugin(pluginUpdateReq *bean2.PluginM return nil, err } } + if len(pluginStepsToUpdate) > 0 { err = impl.updateDeepPluginStepData(pluginStepsToUpdate, pluginStepVariables, pluginStepConditions, pluginSteps, userId, tx) if err != nil { @@ -1909,14 +1910,6 @@ func (impl *GlobalPluginServiceImpl) GetAllPluginMinData() ([]*bean2.PluginMinDt } func (impl *GlobalPluginServiceImpl) validateV2PluginRequest(pluginReq *bean2.PluginParentMetadataDto) error { - if len(pluginReq.Type) == 0 { - return &util.ApiError{ - HttpStatusCode: http.StatusBadRequest, - Code: strconv.Itoa(http.StatusBadRequest), - InternalMessage: bean2.InvalidPluginTypeError, - UserMessage: errors.New(bean2.InvalidPluginTypeError), - } - } if len(pluginReq.Versions.DetailedPluginVersionData) == 0 || pluginReq.Versions.DetailedPluginVersionData[0] == nil { return &util.ApiError{ HttpStatusCode: http.StatusBadRequest, @@ -1930,27 +1923,30 @@ func (impl *GlobalPluginServiceImpl) validateV2PluginRequest(pluginReq *bean2.Pl impl.logger.Errorw("error in getting all plugins", "err", err) return err } - for _, plugin := range plugins { - if plugin.Name == pluginReq.Name { - return &util.ApiError{ - HttpStatusCode: http.StatusConflict, - Code: strconv.Itoa(http.StatusConflict), - InternalMessage: bean2.PluginWithSameNameExistError, - UserMessage: errors.New(bean2.PluginWithSameNameExistError), + if pluginReq.Id == 0 { + //create plugin req. + for _, plugin := range plugins { + if plugin.Identifier == pluginReq.PluginIdentifier { + return &util.ApiError{ + HttpStatusCode: http.StatusConflict, + Code: strconv.Itoa(http.StatusConflict), + InternalMessage: bean2.PluginWithSameIdentifierExistsError, + UserMessage: errors.New(bean2.PluginWithSameIdentifierExistsError), + } } - } - if plugin.Identifier == pluginReq.PluginIdentifier { - return &util.ApiError{ - HttpStatusCode: http.StatusConflict, - Code: strconv.Itoa(http.StatusConflict), - InternalMessage: bean2.PluginWithSameIdentifierExistsError, - UserMessage: errors.New(bean2.PluginWithSameIdentifierExistsError), + if plugin.Name == pluginReq.Name { + return &util.ApiError{ + HttpStatusCode: http.StatusConflict, + Code: strconv.Itoa(http.StatusConflict), + InternalMessage: bean2.PluginWithSameNameExistError, + UserMessage: errors.New(bean2.PluginWithSameNameExistError), + } } } } // semantic versioning validation on plugin's version - pluginVersionDto := pluginReq.Versions.DetailedPluginVersionData[0] - if !semver.IsValid(pluginVersionDto.Version) { + version := fmt.Sprintf("v%s", pluginReq.Versions.DetailedPluginVersionData[0].Version) + if !semver.IsValid(version) { return &util.ApiError{ HttpStatusCode: http.StatusBadRequest, Code: strconv.Itoa(http.StatusBadRequest), @@ -2026,18 +2022,10 @@ func (impl *GlobalPluginServiceImpl) createNewPlugin(tx *pg.Tx, pluginDto *bean2 func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx *pg.Tx, pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { var pluginParentMinData *repository.PluginParentMetadata var err error - if pluginDto.Id > 0 { - pluginParentMinData, err = impl.globalPluginRepository.GetPluginParentMinDataById(pluginDto.Id) - if err != nil { - impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in getting plugin parent metadata", "pluginDto", pluginDto, "err", err) - return 0, err - } - } else { - pluginParentMinData, err = impl.globalPluginRepository.GetPluginParentMinDataByIdentifier(pluginDto.PluginIdentifier) - if err != nil { - impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in getting plugin parent metadata", "pluginDto", pluginDto, "err", err) - return 0, err - } + pluginParentMinData, err = impl.globalPluginRepository.GetPluginParentMinDataById(pluginDto.Id) + if err != nil { + impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in getting plugin parent metadata", "pluginDto", pluginDto, "err", err) + return 0, err } // before saving new plugin version marking previous version's isLatest as false. err = impl.globalPluginRepository.MarkPreviousPluginVersionLatestFalse(pluginParentMinData.Id) @@ -2079,7 +2067,7 @@ func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx * return pluginVersionMetadata.Id, nil } -func (impl *GlobalPluginServiceImpl) createPluginV2(pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { +func (impl *GlobalPluginServiceImpl) CreatePluginOrVersions(pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { err := impl.validateV2PluginRequest(pluginDto) if err != nil { return 0, err @@ -2108,7 +2096,6 @@ func (impl *GlobalPluginServiceImpl) createPluginV2(pluginDto *bean2.PluginParen return 0, err } } - err = tx.Commit() if err != nil { impl.logger.Errorw("createPluginV2, error in committing db transaction", "err", err) @@ -2117,27 +2104,17 @@ func (impl *GlobalPluginServiceImpl) createPluginV2(pluginDto *bean2.PluginParen return versionMetadataId, nil } -func (impl *GlobalPluginServiceImpl) CreatePluginOrVersions(pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { - //pluginMetadataDto := adaptor.GetPluginMetadataDto(pluginDto) - pluginMetadataId, err := impl.createPluginV2(pluginDto, userId) - if err != nil { - impl.logger.Errorw("error in creating plugin", "pluginDto", pluginDto, "err", err) - return 0, err - } - return pluginMetadataId, nil -} - func (impl *GlobalPluginServiceImpl) CreateNewPluginTagsAndRelationsIfRequiredV2(pluginReq *bean2.PluginsVersionDetail, userId int32, tx *pg.Tx) error { allPluginTags, err := impl.globalPluginRepository.GetAllPluginTags() if err != nil { - impl.logger.Errorw("error in getting all plugin tags", "err", err) + impl.logger.Errorw("CreateNewPluginTagsAndRelationsIfRequiredV2, error in getting all plugin tags", "err", err) return err } - existingTagMap := make(map[string]bool, len(allPluginTags)) + existingTagMap := make(map[string]*repository.PluginTag, len(allPluginTags)) for _, tag := range allPluginTags { - existingTagMap[tag.Name] = true + existingTagMap[tag.Name] = tag } - //check for new tags, then create new plugin_tag and plugin_ tag_relation entry in db when new tags are present in request + //check for new tags, then create new plugin_tag and plugin_tag_relation entry in db when new tags are present in request newPluginTagsToCreate := make([]*repository.PluginTag, 0, len(pluginReq.Tags)) newPluginTagRelationsToCreate := make([]*repository.PluginTagRelation, 0, len(pluginReq.Tags)) @@ -2150,19 +2127,22 @@ func (impl *GlobalPluginServiceImpl) CreateNewPluginTagsAndRelationsIfRequiredV2 if len(newPluginTagsToCreate) > 0 { err = impl.globalPluginRepository.SavePluginTagInBulk(newPluginTagsToCreate, tx) if err != nil { - impl.logger.Errorw("error in saving plugin tag", "newPluginTags", newPluginTagsToCreate, "err", err) + impl.logger.Errorw("CreateNewPluginTagsAndRelationsIfRequiredV2, error in saving plugin tag", "newPluginTags", newPluginTagsToCreate, "err", err) return err } + for _, newTag := range newPluginTagsToCreate { + existingTagMap[newTag.Name] = newTag + } } - for _, newTag := range newPluginTagsToCreate { - newPluginTagRelationsToCreate = append(newPluginTagRelationsToCreate, repository.NewPluginTagRelation().CreateAuditLog(userId).WithTagAndPluginId(newTag.Id, pluginReq.Id)) + for _, tag := range pluginReq.Tags { + newPluginTagRelationsToCreate = append(newPluginTagRelationsToCreate, repository.NewPluginTagRelation().CreateAuditLog(userId).WithTagAndPluginId(existingTagMap[tag].Id, pluginReq.Id)) } if len(newPluginTagRelationsToCreate) > 0 { err = impl.globalPluginRepository.SavePluginTagRelationInBulk(newPluginTagRelationsToCreate, tx) if err != nil { - impl.logger.Errorw("error in saving plugin tag relation in bulk", "newPluginTagRelationsToCreate", newPluginTagRelationsToCreate, "err", err) + impl.logger.Errorw("CreateNewPluginTagsAndRelationsIfRequiredV2, error in saving plugin tag relation in bulk", "newPluginTagRelationsToCreate", newPluginTagRelationsToCreate, "err", err) return err } } diff --git a/pkg/plugin/bean/bean.go b/pkg/plugin/bean/bean.go index d531b99e9b..cb26568554 100644 --- a/pkg/plugin/bean/bean.go +++ b/pkg/plugin/bean/bean.go @@ -344,7 +344,7 @@ const ( NoPluginFoundForThisSearchQueryErr = "unable to find desired plugin for the query filter" PluginStepsNotProvidedError = "plugin steps not provided" PluginWithSameNameExistError = "plugin with the same name exists, please choose another name" - PluginWithSameIdentifierExistsError = "plugin with the same identifier exists, please choose another name" + PluginWithSameIdentifierExistsError = "plugin with the same identifier exists, please choose another identifier name" PluginVersionNotSemanticallyCorrectError = "please provide a plugin version that adheres to Semantic Versioning 2.0.0 to ensure compatibility and proper versioning" PluginIconNotCorrectOrReachableError = "cannot validate icon, make sure that provided url link is reachable" InvalidPluginTypeError = "invalid plugin type, should be of the type SHARED" diff --git a/pkg/plugin/repository/GlobalPluginRepository.go b/pkg/plugin/repository/GlobalPluginRepository.go index fcc8ad310b..2929616b96 100644 --- a/pkg/plugin/repository/GlobalPluginRepository.go +++ b/pkg/plugin/repository/GlobalPluginRepository.go @@ -336,7 +336,6 @@ type GlobalPluginRepository interface { GetPluginParentMetadataByIds(ids []int) ([]*PluginParentMetadata, error) GetAllPluginMinData() ([]*PluginParentMetadata, error) GetPluginParentMinDataById(id int) (*PluginParentMetadata, error) - GetPluginParentMinDataByIdentifier(identifier string) (*PluginParentMetadata, error) MarkPreviousPluginVersionLatestFalse(pluginParentId int) error SavePluginMetadata(pluginMetadata *PluginMetadata, tx *pg.Tx) (*PluginMetadata, error) @@ -792,16 +791,6 @@ func (impl *GlobalPluginRepositoryImpl) GetPluginParentMinDataById(id int) (*Plu return &pluginParentMetadata, nil } -func (impl *GlobalPluginRepositoryImpl) GetPluginParentMinDataByIdentifier(identifier string) (*PluginParentMetadata, error) { - var pluginParentMetadata PluginParentMetadata - err := impl.dbConnection.Model(&pluginParentMetadata).Where("identifier = ?", identifier). - Where("deleted = ?", false).Select() - if err != nil { - return nil, err - } - return &pluginParentMetadata, nil -} - func (impl *GlobalPluginRepositoryImpl) SavePluginParentMetadata(tx *pg.Tx, pluginParentMetadata *PluginParentMetadata) (*PluginParentMetadata, error) { err := tx.Insert(pluginParentMetadata) return pluginParentMetadata, err From 40e7365a2d40b885f82c98f8ed2df076a35275a6 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 8 Aug 2024 01:51:12 +0530 Subject: [PATCH 06/23] wip: optimize GetAllFilteredPluginParentMetadata query --- .../repository/GlobalPluginRepository.go | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/pkg/plugin/repository/GlobalPluginRepository.go b/pkg/plugin/repository/GlobalPluginRepository.go index 2929616b96..a39195973c 100644 --- a/pkg/plugin/repository/GlobalPluginRepository.go +++ b/pkg/plugin/repository/GlobalPluginRepository.go @@ -803,24 +803,20 @@ func (impl *GlobalPluginRepositoryImpl) UpdatePluginMetadataInBulk(pluginsMetada func (impl *GlobalPluginRepositoryImpl) GetAllFilteredPluginParentMetadata(searchKey string, tags []string) ([]*PluginParentMetadata, error) { var plugins []*PluginParentMetadata - subQuery := "select ppm.id, ppm.identifier,ppm.name,ppm.description,ppm.type,ppm.icon,ppm.deleted,ppm.created_by, ppm.created_on,ppm.updated_by,ppm.updated_on from plugin_parent_metadata ppm" + + query := "select ppm.id, ppm.identifier,ppm.name,ppm.description,ppm.type,ppm.icon,ppm.deleted,ppm.created_by, ppm.created_on,ppm.updated_by,ppm.updated_on from plugin_parent_metadata ppm" + " inner join plugin_metadata pm on pm.plugin_parent_metadata_id=ppm.id" - whereCondition := fmt.Sprintf(" where ppm.deleted=false") - orderCondition := fmt.Sprintf(" ORDER BY ppm.id asc") + whereCondition := fmt.Sprintf(" where ppm.deleted=false AND pm.deleted=false AND pm.is_latest=true") if len(tags) > 0 { - subQuery = "select DISTINCT ON(ppm.id) ppm.id, ppm.identifier,ppm.name,ppm.description,ppm.type,ppm.icon,ppm.deleted,ppm.created_by, ppm.created_on,ppm.updated_by,ppm.updated_on from plugin_parent_metadata ppm" + - " inner join plugin_metadata pm on pm.plugin_parent_metadata_id=ppm.id" + - " left join plugin_tag_relation ptr on ptr.plugin_id=pm.id" + - " left join plugin_tag pt on ptr.tag_id=pt.id" - whereCondition += fmt.Sprintf(" AND pm.deleted=false AND pt.deleted=false AND pt.name in (%s)", helper.GetCommaSepratedStringWithComma(tags)) + tagFilterSubQuery := fmt.Sprintf("select ptr.plugin_id from plugin_tag_relation ptr inner join plugin_tag pt on ptr.tag_id =pt.id where pt.deleted =false and pt.name in (%s) group by ptr.plugin_id having count(ptr.plugin_id )=%d", helper.GetCommaSepratedStringWithComma(tags), len(tags)) + whereCondition += fmt.Sprintf(" AND pm.id in (%s)", tagFilterSubQuery) } if len(searchKey) > 0 { searchKeyLike := "%" + searchKey + "%" - whereCondition += fmt.Sprintf(" AND (pm.description ilike '%s' or pm.name ilike '%s')", searchKeyLike, searchKeyLike) + whereCondition += fmt.Sprintf(" AND (ppm.description ilike '%s' or ppm.name ilike '%s')", searchKeyLike, searchKeyLike) } - whereCondition += fmt.Sprintf(" AND pm.is_latest=true") - subQuery += whereCondition + orderCondition - query := fmt.Sprintf(" select * from (%s) x ORDER BY name asc;", subQuery) + orderCondition := " ORDER BY ppm.name asc;" + + query += whereCondition + orderCondition _, err := impl.dbConnection.Query(&plugins, query) if err != nil { return nil, err From 638a5e76eb592539bbb0cac519dcf085d321653b Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 8 Aug 2024 11:45:59 +0530 Subject: [PATCH 07/23] wip: create plugin tag new flow --- pkg/plugin/GlobalPluginService.go | 47 ++++++++++++++++--- .../repository/GlobalPluginRepository.go | 14 ++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index 719690c6ef..3b47c88ecc 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -2007,16 +2007,51 @@ func (impl *GlobalPluginServiceImpl) createNewPlugin(tx *pg.Tx, pluginDto *bean2 return 0, util.NewApiError().WithCode(strconv.Itoa(http.StatusBadRequest)).WithHttpStatusCode(http.StatusBadRequest). WithInternalMessage(bean2.PluginStepsNotProvidedError).WithUserMessage(errors.New(bean2.PluginStepsNotProvidedError)) } - newTagsPresentInReq := pluginDto.Versions.DetailedPluginVersionData[0].NewTagsPresent - if newTagsPresentInReq { - err = impl.CreateNewPluginTagsAndRelationsIfRequiredV2(pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) + + err = impl.createPluginTagAndRelations(pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in createPluginTagAndRelations", "tags", pluginDto.Versions.DetailedPluginVersionData[0].Tags, "err", err) + return 0, err + } + return pluginVersionMetadata.Id, nil +} + +func (impl *GlobalPluginServiceImpl) createPluginTagAndRelations(pluginReq *bean2.PluginsVersionDetail, userId int32, tx *pg.Tx) error { + if pluginReq.NewTagsPresent { + err := impl.CreateNewPluginTagsAndRelationsIfRequiredV2(pluginReq, userId, tx) if err != nil { - impl.logger.Errorw("createNewPlugin, error in CreateNewPluginTagsAndRelationsIfRequired", "tags", pluginDto.Versions.DetailedPluginVersionData[0].Tags, "err", err) - return 0, err + impl.logger.Errorw("createPluginTagAndRelations, error in CreateNewPluginTagsAndRelationsIfRequired", "tags", pluginReq.Tags, "err", err) + return err + } + } else { + err := impl.CreatePluginTagRelations(pluginReq, userId, tx) + if err != nil { + impl.logger.Errorw("createPluginTagAndRelations, error in CreatePluginTagRelations", "tags", pluginReq.Tags, "err", err) + return err } } + return nil +} - return pluginVersionMetadata.Id, nil +func (impl *GlobalPluginServiceImpl) CreatePluginTagRelations(pluginReq *bean2.PluginsVersionDetail, userId int32, tx *pg.Tx) error { + tags, err := impl.globalPluginRepository.GetPluginTagByNames(pluginReq.Tags) + if err != nil { + impl.logger.Errorw("CreatePluginTagRelations, error in GetPluginTagByNames", "tags", pluginReq.Tags, "err", err) + return err + } + newPluginTagRelationsToCreate := make([]*repository.PluginTagRelation, 0, len(pluginReq.Tags)) + for _, tag := range tags { + newPluginTagRelationsToCreate = append(newPluginTagRelationsToCreate, repository.NewPluginTagRelation().CreateAuditLog(userId).WithTagAndPluginId(tag.Id, pluginReq.Id)) + } + + if len(newPluginTagRelationsToCreate) > 0 { + err = impl.globalPluginRepository.SavePluginTagRelationInBulk(newPluginTagRelationsToCreate, tx) + if err != nil { + impl.logger.Errorw("CreatePluginTagRelations, error in saving plugin tag relation in bulk", "newPluginTagRelationsToCreate", newPluginTagRelationsToCreate, "err", err) + return err + } + } + return nil } func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx *pg.Tx, pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { diff --git a/pkg/plugin/repository/GlobalPluginRepository.go b/pkg/plugin/repository/GlobalPluginRepository.go index a39195973c..4773eb5fb5 100644 --- a/pkg/plugin/repository/GlobalPluginRepository.go +++ b/pkg/plugin/repository/GlobalPluginRepository.go @@ -314,6 +314,7 @@ type GlobalPluginRepository interface { GetMetaDataByPluginId(pluginId int) (*PluginMetadata, error) GetMetaDataByPluginIds(pluginIds []int) ([]*PluginMetadata, error) GetAllPluginTags() ([]*PluginTag, error) + GetPluginTagByNames(tagNames []string) ([]*PluginTag, error) GetAllPluginTagRelations() ([]*PluginTagRelation, error) GetTagsByPluginId(pluginId int) ([]string, error) GetScriptDetailById(id int) (*PluginPipelineScript, error) @@ -420,6 +421,19 @@ func (impl *GlobalPluginRepositoryImpl) GetAllPluginTags() ([]*PluginTag, error) return tags, nil } +func (impl *GlobalPluginRepositoryImpl) GetPluginTagByNames(tagNames []string) ([]*PluginTag, error) { + var tags []*PluginTag + err := impl.dbConnection.Model(&tags). + Where("deleted = ?", false). + Where("name in (?)", pg.In(tagNames)). + Select() + if err != nil { + impl.logger.Errorw("err in getting all tags by names", "tagNames", tagNames, "err", err) + return nil, err + } + return tags, nil +} + func (impl *GlobalPluginRepositoryImpl) GetAllPluginTagRelations() ([]*PluginTagRelation, error) { var rel []*PluginTagRelation err := impl.dbConnection.Model(&rel). From db878aec6f7fcdb4f86cbe72ef917843a8de540d Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 8 Aug 2024 11:46:26 +0530 Subject: [PATCH 08/23] wip: minor fix --- pkg/plugin/GlobalPluginService.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index 3b47c88ecc..c67f39bfcc 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -2090,13 +2090,11 @@ func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx * return 0, util.NewApiError().WithCode(strconv.Itoa(http.StatusBadRequest)).WithHttpStatusCode(http.StatusBadRequest). WithInternalMessage(bean2.PluginStepsNotProvidedError).WithUserMessage(errors.New(bean2.PluginStepsNotProvidedError)) } - newTagsPresentInReq := pluginDto.Versions.DetailedPluginVersionData[0].NewTagsPresent - if newTagsPresentInReq { - err = impl.CreateNewPluginTagsAndRelationsIfRequiredV2(pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) - if err != nil { - impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in CreateNewPluginTagsAndRelationsIfRequired", "tags", pluginDto.Versions.DetailedPluginVersionData[0].Tags, "err", err) - return 0, err - } + + err = impl.createPluginTagAndRelations(pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in createPluginTagAndRelations", "tags", pluginDto.Versions.DetailedPluginVersionData[0].Tags, "err", err) + return 0, err } return pluginVersionMetadata.Id, nil From 046fb83410777acd0b6fc2128f9586dea9094c40 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 8 Aug 2024 11:50:55 +0530 Subject: [PATCH 09/23] wip: minor fix --- pkg/plugin/repository/GlobalPluginRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/repository/GlobalPluginRepository.go b/pkg/plugin/repository/GlobalPluginRepository.go index 4773eb5fb5..656aa5c18a 100644 --- a/pkg/plugin/repository/GlobalPluginRepository.go +++ b/pkg/plugin/repository/GlobalPluginRepository.go @@ -826,7 +826,7 @@ func (impl *GlobalPluginRepositoryImpl) GetAllFilteredPluginParentMetadata(searc } if len(searchKey) > 0 { searchKeyLike := "%" + searchKey + "%" - whereCondition += fmt.Sprintf(" AND (ppm.description ilike '%s' or ppm.name ilike '%s')", searchKeyLike, searchKeyLike) + whereCondition += fmt.Sprintf(" AND (pm.description ilike '%s' or pm.name ilike '%s')", searchKeyLike, searchKeyLike) } orderCondition := " ORDER BY ppm.name asc;" From 309705fc0d147075b8af1fbc6d71f6a194ab4060 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 8 Aug 2024 12:36:28 +0530 Subject: [PATCH 10/23] wip: minor fix --- pkg/plugin/GlobalPluginService.go | 1 + pkg/plugin/repository/GlobalPluginRepository.go | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index c67f39bfcc..b662ec50db 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -2068,6 +2068,7 @@ func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx * impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in MarkPreviousPluginVersionLatestFalse", "pluginParentId", pluginDto.Id, "err", err) return 0, err } + pluginDto.Name = pluginParentMinData.Name pluginVersionDto := adaptor.GetPluginVersionMetadataDbObject(pluginDto, userId). WithPluginParentMetadataId(pluginParentMinData.Id). WithIsLatestFlag(true) diff --git a/pkg/plugin/repository/GlobalPluginRepository.go b/pkg/plugin/repository/GlobalPluginRepository.go index 656aa5c18a..d2330d8ade 100644 --- a/pkg/plugin/repository/GlobalPluginRepository.go +++ b/pkg/plugin/repository/GlobalPluginRepository.go @@ -797,7 +797,9 @@ func (impl *GlobalPluginRepositoryImpl) GetPluginParentMetadataByIdentifier(plug func (impl *GlobalPluginRepositoryImpl) GetPluginParentMinDataById(id int) (*PluginParentMetadata, error) { var pluginParentMetadata PluginParentMetadata - err := impl.dbConnection.Model(&pluginParentMetadata).Where("id = ?", id). + err := impl.dbConnection.Model(&pluginParentMetadata). + Column("plugin_parent_metadata.id", "plugin_parent_metadata.name"). + Where("id = ?", id). Where("deleted = ?", false).Select() if err != nil { return nil, err From eff0501c0756d00b91b00b2a38cb3473b557f314 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 8 Aug 2024 12:42:32 +0530 Subject: [PATCH 11/23] wip: newTagsPresent -> areNewTagsPresent --- pkg/plugin/bean/bean.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/bean/bean.go b/pkg/plugin/bean/bean.go index cb26568554..121b5439ae 100644 --- a/pkg/plugin/bean/bean.go +++ b/pkg/plugin/bean/bean.go @@ -41,7 +41,7 @@ type PluginMetadataDto struct { Action int `json:"action,omitempty"` PluginStage string `json:"pluginStage,omitempty"` PluginSteps []*PluginStepsDto `json:"pluginSteps,omitempty"` - NewTagsPresent bool `json:"newTagsPresent,omitempty"` + NewTagsPresent bool `json:"areNewTagsPresent,omitempty"` //PluginIdentifier string `json:"pluginIdentifier" validate:"required,min=3,max=100,global-entity-name"` } From f43da65953378ac3d779c6f159a603d3dee60da2 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 8 Aug 2024 15:12:57 +0530 Subject: [PATCH 12/23] wip: icon is not mandatory code incorporated --- pkg/plugin/GlobalPluginService.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index b662ec50db..4901b528fb 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -1955,13 +1955,15 @@ func (impl *GlobalPluginServiceImpl) validateV2PluginRequest(pluginReq *bean2.Pl } } //validate icon url and size - err = utils.FetchIconAndCheckSize(pluginReq.Icon, bean2.PluginIconMaxSizeInBytes) - if err != nil { - return &util.ApiError{ - HttpStatusCode: http.StatusBadRequest, - Code: strconv.Itoa(http.StatusBadRequest), - InternalMessage: fmt.Sprintf("%s err:= %s", bean2.PluginIconNotCorrectOrReachableError, err.Error()), - UserMessage: errors.New(fmt.Sprintf("%s err:= %s", bean2.PluginIconNotCorrectOrReachableError, err.Error())), + if len(pluginReq.Icon) > 0 { + err = utils.FetchIconAndCheckSize(pluginReq.Icon, bean2.PluginIconMaxSizeInBytes) + if err != nil { + return &util.ApiError{ + HttpStatusCode: http.StatusBadRequest, + Code: strconv.Itoa(http.StatusBadRequest), + InternalMessage: fmt.Sprintf("%s err:= %s", bean2.PluginIconNotCorrectOrReachableError, err.Error()), + UserMessage: fmt.Sprintf("%s err:= %s", bean2.PluginIconNotCorrectOrReachableError, err.Error()), + } } } return nil From d906333861306fbb15bd854d8da7b9c0ee838abc Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 8 Aug 2024 15:24:10 +0530 Subject: [PATCH 13/23] wip:minor refactoring --- api/restHandler/GlobalPluginRestHandler.go | 3 +- pkg/plugin/GlobalPluginService.go | 2 +- pkg/plugin/bean/bean.go | 33 +++++++++++++--------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/api/restHandler/GlobalPluginRestHandler.go b/api/restHandler/GlobalPluginRestHandler.go index b08c38eb4f..5764afcd90 100644 --- a/api/restHandler/GlobalPluginRestHandler.go +++ b/api/restHandler/GlobalPluginRestHandler.go @@ -452,7 +452,8 @@ func (handler *GlobalPluginRestHandlerImpl) CreatePlugin(w http.ResponseWriter, common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - common.WriteJsonResp(w, nil, pluginVersionId, http.StatusOK) + + common.WriteJsonResp(w, nil, bean.NewPluginMinDto().WithPluginVersionId(pluginVersionId), http.StatusOK) } func (handler *GlobalPluginRestHandlerImpl) GetAllPluginMinData(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index 4901b528fb..a8b40ff423 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -2019,7 +2019,7 @@ func (impl *GlobalPluginServiceImpl) createNewPlugin(tx *pg.Tx, pluginDto *bean2 } func (impl *GlobalPluginServiceImpl) createPluginTagAndRelations(pluginReq *bean2.PluginsVersionDetail, userId int32, tx *pg.Tx) error { - if pluginReq.NewTagsPresent { + if pluginReq.AreNewTagsPresent { err := impl.CreateNewPluginTagsAndRelationsIfRequiredV2(pluginReq, userId, tx) if err != nil { impl.logger.Errorw("createPluginTagAndRelations, error in CreateNewPluginTagsAndRelationsIfRequired", "tags", pluginReq.Tags, "err", err) diff --git a/pkg/plugin/bean/bean.go b/pkg/plugin/bean/bean.go index 121b5439ae..382b7ee4db 100644 --- a/pkg/plugin/bean/bean.go +++ b/pkg/plugin/bean/bean.go @@ -32,23 +32,23 @@ type PluginListComponentDto struct { //created new struct for backward compatibi } type PluginMetadataDto struct { - Id int `json:"id"` - Name string `json:"name" validate:"required,min=3,max=100,global-entity-name"` - Description string `json:"description" validate:"max=300"` - Type string `json:"type,omitempty" validate:"oneof=SHARED PRESET"` // SHARED, PRESET etc - Icon string `json:"icon,omitempty"` - Tags []string `json:"tags"` - Action int `json:"action,omitempty"` - PluginStage string `json:"pluginStage,omitempty"` - PluginSteps []*PluginStepsDto `json:"pluginSteps,omitempty"` - NewTagsPresent bool `json:"areNewTagsPresent,omitempty"` - //PluginIdentifier string `json:"pluginIdentifier" validate:"required,min=3,max=100,global-entity-name"` + Id int `json:"id"` + Name string `json:"name" validate:"required,min=3,max=100,global-entity-name"` + Description string `json:"description" validate:"max=300"` + Type string `json:"type,omitempty" validate:"oneof=SHARED PRESET"` // SHARED, PRESET etc + Icon string `json:"icon,omitempty"` + Tags []string `json:"tags"` + Action int `json:"action,omitempty"` + PluginStage string `json:"pluginStage,omitempty"` + PluginSteps []*PluginStepsDto `json:"pluginSteps,omitempty"` + AreNewTagsPresent bool `json:"areNewTagsPresent,omitempty"` } type PluginMinDto struct { - ParentPluginId int `json:"parentPluginId"` - PluginName string `json:"pluginName"` - Icon string `json:"icon"` + ParentPluginId int `json:"id,omitempty"` + PluginName string `json:"name,omitempty"` + Icon string `json:"icon,omitempty"` + PluginVersionId int `json:"pluginVersionId,omitempty"` } func NewPluginMinDto() *PluginMinDto { @@ -70,6 +70,11 @@ func (r *PluginMinDto) WithIcon(icon string) *PluginMinDto { return r } +func (r *PluginMinDto) WithPluginVersionId(versionId int) *PluginMinDto { + r.PluginVersionId = versionId + return r +} + type PluginsDto struct { ParentPlugins []*PluginParentMetadataDto `json:"parentPlugins"` TotalCount int `json:"totalCount"` From ff188ebba7489f708212ad8e26718320cc8dcd97 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 8 Aug 2024 17:19:43 +0530 Subject: [PATCH 14/23] wip: prevent duplicate version from being created and save tags relation only when --- pkg/plugin/GlobalPluginService.go | 89 +++++++++++++------ pkg/plugin/bean/bean.go | 2 +- .../repository/GlobalPluginRepository.go | 15 ++++ 3 files changed, 78 insertions(+), 28 deletions(-) diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index a8b40ff423..7e094728fd 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -1909,54 +1909,89 @@ func (impl *GlobalPluginServiceImpl) GetAllPluginMinData() ([]*bean2.PluginMinDt return pluginMinList, nil } +func (impl *GlobalPluginServiceImpl) checkValidationOnPluginNameAndIdentifier(pluginReq *bean2.PluginParentMetadataDto) error { + plugins, err := impl.globalPluginRepository.GetAllPluginMinData() + if err != nil { + impl.logger.Errorw("error in getting all plugins", "err", err) + return err + } + for _, plugin := range plugins { + if plugin.Identifier == pluginReq.PluginIdentifier { + return &util.ApiError{ + HttpStatusCode: http.StatusConflict, + Code: strconv.Itoa(http.StatusConflict), + InternalMessage: bean2.PluginWithSameIdentifierExistsError, + UserMessage: bean2.PluginWithSameIdentifierExistsError, + } + } + if plugin.Name == pluginReq.Name { + return &util.ApiError{ + HttpStatusCode: http.StatusConflict, + Code: strconv.Itoa(http.StatusConflict), + InternalMessage: bean2.PluginWithSameNameExistError, + UserMessage: bean2.PluginWithSameNameExistError, + } + } + } + return nil +} + +func (impl *GlobalPluginServiceImpl) checkValidationOnVersion(pluginReq *bean2.PluginParentMetadataDto) error { + pluginVersions, err := impl.globalPluginRepository.GetPluginVersionsByParentId(pluginReq.Id) + if err != nil { + impl.logger.Errorw("checkValidationOnVersion, error in getting all plugins versions by parentPluginId", "parentPluginId", pluginReq.Id, "err", err) + return err + } + for _, pluginVersion := range pluginVersions { + if pluginVersion.PluginVersion == pluginReq.Versions.DetailedPluginVersionData[0].Version { + return &util.ApiError{ + HttpStatusCode: http.StatusBadRequest, + Code: strconv.Itoa(http.StatusBadRequest), + InternalMessage: bean2.PluginVersionAlreadyExistError, + UserMessage: bean2.PluginVersionAlreadyExistError, + } + } + } + return nil +} + func (impl *GlobalPluginServiceImpl) validateV2PluginRequest(pluginReq *bean2.PluginParentMetadataDto) error { if len(pluginReq.Versions.DetailedPluginVersionData) == 0 || pluginReq.Versions.DetailedPluginVersionData[0] == nil { return &util.ApiError{ HttpStatusCode: http.StatusBadRequest, Code: strconv.Itoa(http.StatusBadRequest), InternalMessage: bean2.NoStepDataToProceedError, - UserMessage: errors.New(bean2.NoStepDataToProceedError), + UserMessage: bean2.NoStepDataToProceedError, } } - plugins, err := impl.globalPluginRepository.GetAllPluginMinData() - if err != nil { - impl.logger.Errorw("error in getting all plugins", "err", err) - return err - } if pluginReq.Id == 0 { //create plugin req. - for _, plugin := range plugins { - if plugin.Identifier == pluginReq.PluginIdentifier { - return &util.ApiError{ - HttpStatusCode: http.StatusConflict, - Code: strconv.Itoa(http.StatusConflict), - InternalMessage: bean2.PluginWithSameIdentifierExistsError, - UserMessage: errors.New(bean2.PluginWithSameIdentifierExistsError), - } - } - if plugin.Name == pluginReq.Name { - return &util.ApiError{ - HttpStatusCode: http.StatusConflict, - Code: strconv.Itoa(http.StatusConflict), - InternalMessage: bean2.PluginWithSameNameExistError, - UserMessage: errors.New(bean2.PluginWithSameNameExistError), - } - } + err := impl.checkValidationOnPluginNameAndIdentifier(pluginReq) + if err != nil { + impl.logger.Errorw("error in checkValidationOnPluginNameAndIdentifier", "err", err) + return err + } + } else { + err := impl.checkValidationOnVersion(pluginReq) + if err != nil { + impl.logger.Errorw("error in checkValidationOnPluginNameAndIdentifier", "err", err) + return err } } - // semantic versioning validation on plugin's version + version := fmt.Sprintf("v%s", pluginReq.Versions.DetailedPluginVersionData[0].Version) + // semantic versioning validation on plugin's version if !semver.IsValid(version) { return &util.ApiError{ HttpStatusCode: http.StatusBadRequest, Code: strconv.Itoa(http.StatusBadRequest), InternalMessage: bean2.PluginVersionNotSemanticallyCorrectError, - UserMessage: errors.New(bean2.PluginVersionNotSemanticallyCorrectError), + UserMessage: bean2.PluginVersionNotSemanticallyCorrectError, } } //validate icon url and size if len(pluginReq.Icon) > 0 { - err = utils.FetchIconAndCheckSize(pluginReq.Icon, bean2.PluginIconMaxSizeInBytes) + err := utils.FetchIconAndCheckSize(pluginReq.Icon, bean2.PluginIconMaxSizeInBytes) if err != nil { return &util.ApiError{ HttpStatusCode: http.StatusBadRequest, @@ -2025,7 +2060,7 @@ func (impl *GlobalPluginServiceImpl) createPluginTagAndRelations(pluginReq *bean impl.logger.Errorw("createPluginTagAndRelations, error in CreateNewPluginTagsAndRelationsIfRequired", "tags", pluginReq.Tags, "err", err) return err } - } else { + } else if len(pluginReq.Tags) > 0 { err := impl.CreatePluginTagRelations(pluginReq, userId, tx) if err != nil { impl.logger.Errorw("createPluginTagAndRelations, error in CreatePluginTagRelations", "tags", pluginReq.Tags, "err", err) diff --git a/pkg/plugin/bean/bean.go b/pkg/plugin/bean/bean.go index 382b7ee4db..5f65320a18 100644 --- a/pkg/plugin/bean/bean.go +++ b/pkg/plugin/bean/bean.go @@ -352,7 +352,7 @@ const ( PluginWithSameIdentifierExistsError = "plugin with the same identifier exists, please choose another identifier name" PluginVersionNotSemanticallyCorrectError = "please provide a plugin version that adheres to Semantic Versioning 2.0.0 to ensure compatibility and proper versioning" PluginIconNotCorrectOrReachableError = "cannot validate icon, make sure that provided url link is reachable" - InvalidPluginTypeError = "invalid plugin type, should be of the type SHARED" + PluginVersionAlreadyExistError = "this plugin version already exists, please provide another plugin version" NoStepDataToProceedError = "no step data provided to save, please provide a plugin step to proceed further" ) diff --git a/pkg/plugin/repository/GlobalPluginRepository.go b/pkg/plugin/repository/GlobalPluginRepository.go index d2330d8ade..147f6fb725 100644 --- a/pkg/plugin/repository/GlobalPluginRepository.go +++ b/pkg/plugin/repository/GlobalPluginRepository.go @@ -331,6 +331,7 @@ type GlobalPluginRepository interface { GetConditionsByPluginId(pluginId int) ([]*PluginStepCondition, error) GetPluginStageMappingByPluginId(pluginId int) (*PluginStageMapping, error) GetConnection() (dbConnection *pg.DB) + GetPluginVersionsByParentId(parentPluginId int) ([]*PluginMetadata, error) GetPluginParentMetadataByIdentifier(pluginIdentifier string) (*PluginParentMetadata, error) GetAllFilteredPluginParentMetadata(searchKey string, tags []string) ([]*PluginParentMetadata, error) @@ -606,6 +607,20 @@ func (impl *GlobalPluginRepositoryImpl) GetPluginByName(pluginName string) ([]*P } +func (impl *GlobalPluginRepositoryImpl) GetPluginVersionsByParentId(parentPluginId int) ([]*PluginMetadata, error) { + var plugin []*PluginMetadata + err := impl.dbConnection.Model(&plugin). + Where("plugin_parent_metadata_id = ?", parentPluginId). + Where("deleted = ?", false). + Where("is_deprecated = ?", false). + Select() + if err != nil { + impl.logger.Errorw("err in getting pluginVersionMetadata by parentPluginId", "parentPluginId", parentPluginId, "err", err) + return nil, err + } + return plugin, nil +} + func (impl *GlobalPluginRepositoryImpl) GetAllPluginMetaData() ([]*PluginMetadata, error) { var plugins []*PluginMetadata err := impl.dbConnection.Model(&plugins).Where("deleted = ?", false).Select() From 19e66a3ce291320687fba21d9a9b934d2695e471 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 8 Aug 2024 17:33:09 +0530 Subject: [PATCH 15/23] wip: minor fix --- pkg/plugin/adaptor/adaptor.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/plugin/adaptor/adaptor.go b/pkg/plugin/adaptor/adaptor.go index c26c5abecb..bbd95706f5 100644 --- a/pkg/plugin/adaptor/adaptor.go +++ b/pkg/plugin/adaptor/adaptor.go @@ -16,12 +16,12 @@ func GetPluginVersionMetadataDbObject(pluginDto *bean2.PluginParentMetadataDto, return repository.NewPluginVersionMetadata().CreateAuditLog(userId).WithBasicMetadata(pluginDto.Name, versionDto.Description, versionDto.Version, versionDto.DocLink) } -func GetPluginStepDbObject(pluginStepDto *bean2.PluginStepsDto, pluginVersionMetadata int, userId int32) *repository.PluginStep { +func GetPluginStepDbObject(pluginStepDto *bean2.PluginStepsDto, pluginVersionMetadataId int, userId int32) *repository.PluginStep { return &repository.PluginStep{ - PluginId: pluginVersionMetadata, - Name: pluginStepDto.Name, + PluginId: pluginVersionMetadataId, + Name: "Step-1", Description: pluginStepDto.Description, - Index: pluginStepDto.Index, + Index: 1, StepType: pluginStepDto.StepType, RefPluginId: pluginStepDto.RefPluginId, OutputDirectoryPath: pluginStepDto.OutputDirectoryPath, From c2e7cdeb32f6fb32f0c7c3d6ccd116b3594225e5 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 8 Aug 2024 19:33:37 +0530 Subject: [PATCH 16/23] wip: details api, get all plugin data or non --- pkg/plugin/GlobalPluginService.go | 50 +++++++++++++++++++++++++++---- pkg/plugin/adaptor/adaptor.go | 2 +- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index 7e094728fd..cc089d379f 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -1750,24 +1750,62 @@ func (impl *GlobalPluginServiceImpl) ListAllPluginsV2(filter *bean2.PluginsListF return pluginDetails, nil } +func (impl *GlobalPluginServiceImpl) validateDetailRequest(pluginVersions []*repository.PluginMetadata, pluginVersionIds, parentPluginIds []int) error { + pluginVersionsIdMap, pluginParentIdMap := make(map[int]bool, len(pluginVersionIds)), make(map[int]bool, len(parentPluginIds)) + allPlugins, err := impl.globalPluginRepository.GetAllPluginMinData() + if err != nil { + impl.logger.Errorw("validateDetailRequest, error in getting all plugins parent metadata", "err", err) + return err + } + for _, pluginVersion := range pluginVersions { + pluginVersionsIdMap[pluginVersion.Id] = true + } + for _, plugin := range allPlugins { + pluginParentIdMap[plugin.Id] = true + } + nonExistingPluginVersionIds, nonExistingPluginParentIds := make([]int, 0, len(pluginVersionIds)), make([]int, 0, len(parentPluginIds)) + for _, versionId := range pluginVersionIds { + if _, ok := pluginVersionsIdMap[versionId]; !ok { + nonExistingPluginVersionIds = append(nonExistingPluginVersionIds, versionId) + } + } + for _, pluginId := range parentPluginIds { + if _, ok := pluginParentIdMap[pluginId]; !ok { + nonExistingPluginParentIds = append(nonExistingPluginParentIds, pluginId) + } + } + if len(nonExistingPluginVersionIds) > 0 { + errorMsg := fmt.Sprintf("there are some plugin version ids in request that do not exist %v", nonExistingPluginVersionIds) + return util.NewApiError().WithHttpStatusCode(http.StatusBadRequest).WithCode(strconv.Itoa(http.StatusBadRequest)).WithUserMessage(errorMsg).WithInternalMessage(errorMsg) + } + if len(nonExistingPluginParentIds) > 0 { + errorMsg := fmt.Sprintf("there are some plugin parent ids in request that do not exist %v", nonExistingPluginParentIds) + return util.NewApiError().WithHttpStatusCode(http.StatusBadRequest).WithCode(strconv.Itoa(http.StatusBadRequest)).WithUserMessage(errorMsg).WithInternalMessage(errorMsg) + } + return nil +} // GetPluginDetailV2 returns all details of the of a plugin version according to the pluginVersionIds and parentPluginIds // provided by user, and minimal data for all versions of that plugin. func (impl *GlobalPluginServiceImpl) GetPluginDetailV2(pluginVersionIds, parentPluginIds []int, fetchAllVersionDetails bool) (*bean2.PluginsDto, error) { + var err error + pluginVersionsMetadata, err := impl.globalPluginRepository.GetMetaDataForAllPlugins() + if err != nil { + impl.logger.Errorw("GetPluginDetailV2, error in getting all plugins versions metadata", "err", err) + return nil, err + } + err = impl.validateDetailRequest(pluginVersionsMetadata, pluginVersionIds, parentPluginIds) + if err != nil { + return nil, err + } pluginParentMetadataDtos := make([]*bean2.PluginParentMetadataDto, 0, len(pluginVersionIds)+len(parentPluginIds)) if len(pluginVersionIds) == 0 && len(parentPluginIds) == 0 { return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: strconv.Itoa(http.StatusBadRequest), InternalMessage: bean2.NoPluginOrParentIdProvidedErr, UserMessage: bean2.NoPluginOrParentIdProvidedErr} } pluginVersionIdsMap, parentPluginIdsMap := helper2.GetPluginVersionAndParentPluginIdsMap(pluginVersionIds, parentPluginIds) - var err error pluginParentMetadataIds := make([]int, 0, len(pluginVersionIds)+len(parentPluginIds)) pluginVersionsIdToInclude := make(map[int]bool, len(pluginVersionIds)+len(parentPluginIds)) - pluginVersionsMetadata, err := impl.globalPluginRepository.GetMetaDataForAllPlugins() - if err != nil { - impl.logger.Errorw("GetPluginDetailV2, error in getting all plugins versions metadata", "err", err) - return nil, err - } filteredPluginVersionMetadata := helper2.GetPluginVersionsMetadataByVersionAndParentPluginIds(pluginVersionsMetadata, pluginVersionIdsMap, parentPluginIdsMap) if len(filteredPluginVersionMetadata) == 0 { diff --git a/pkg/plugin/adaptor/adaptor.go b/pkg/plugin/adaptor/adaptor.go index bbd95706f5..c61dea86ff 100644 --- a/pkg/plugin/adaptor/adaptor.go +++ b/pkg/plugin/adaptor/adaptor.go @@ -19,7 +19,7 @@ func GetPluginVersionMetadataDbObject(pluginDto *bean2.PluginParentMetadataDto, func GetPluginStepDbObject(pluginStepDto *bean2.PluginStepsDto, pluginVersionMetadataId int, userId int32) *repository.PluginStep { return &repository.PluginStep{ PluginId: pluginVersionMetadataId, - Name: "Step-1", + Name: pluginStepDto.Name, Description: pluginStepDto.Description, Index: 1, StepType: pluginStepDto.StepType, From 8dc73866c5a0cfce74a47170ead2ef82afde5b60 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 9 Aug 2024 10:29:49 +0530 Subject: [PATCH 17/23] wip: code review incorp part -1 --- api/restHandler/GlobalPluginRestHandler.go | 4 +- internal/util/ErrorUtil.go | 9 ++ pkg/plugin/GlobalPluginService.go | 96 ++++++++----------- pkg/plugin/bean/bean.go | 13 ++- .../repository/GlobalPluginRepository.go | 14 ++- 5 files changed, 74 insertions(+), 62 deletions(-) diff --git a/api/restHandler/GlobalPluginRestHandler.go b/api/restHandler/GlobalPluginRestHandler.go index 5764afcd90..21d225e1b6 100644 --- a/api/restHandler/GlobalPluginRestHandler.go +++ b/api/restHandler/GlobalPluginRestHandler.go @@ -424,14 +424,14 @@ func (handler *GlobalPluginRestHandlerImpl) MigratePluginData(w http.ResponseWri } func (handler *GlobalPluginRestHandlerImpl) CreatePlugin(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) userId, err := handler.userService.GetLoggedInUser(r) if userId == 0 || err != nil { common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } + decoder := json.NewDecoder(r.Body) var pluginDataDto *bean.PluginParentMetadataDto - err = decoder.Decode(&pluginDataDto) + err = decoder.Decode(pluginDataDto) if err != nil { handler.logger.Errorw("request err, CreatePlugin", "error", err, "payload", pluginDataDto) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) diff --git a/internal/util/ErrorUtil.go b/internal/util/ErrorUtil.go index 59a8d23441..43f3c9b942 100644 --- a/internal/util/ErrorUtil.go +++ b/internal/util/ErrorUtil.go @@ -26,6 +26,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "net/http" + "strconv" ) type ApiError struct { @@ -36,6 +37,14 @@ type ApiError struct { UserDetailMessage string `json:"userDetailMessage,omitempty"` } +func GetApiError(code int, userMessage, internalMessage string) *ApiError { + return &ApiError{ + HttpStatusCode: code, + Code: strconv.Itoa(code), + InternalMessage: internalMessage, + UserMessage: userMessage, + } +} func NewApiError() *ApiError { return &ApiError{} } diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index cc089d379f..2a8609f9a7 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -1,5 +1,17 @@ /* * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package plugin @@ -22,7 +34,6 @@ import ( "go.uber.org/zap" "golang.org/x/mod/semver" "net/http" - "strconv" "strings" "time" ) @@ -1763,25 +1774,18 @@ func (impl *GlobalPluginServiceImpl) validateDetailRequest(pluginVersions []*rep for _, plugin := range allPlugins { pluginParentIdMap[plugin.Id] = true } - nonExistingPluginVersionIds, nonExistingPluginParentIds := make([]int, 0, len(pluginVersionIds)), make([]int, 0, len(parentPluginIds)) for _, versionId := range pluginVersionIds { if _, ok := pluginVersionsIdMap[versionId]; !ok { - nonExistingPluginVersionIds = append(nonExistingPluginVersionIds, versionId) + errorMsg := fmt.Sprintf("there are some plugin version ids in request that do not exist:- %d", versionId) + return util.GetApiError(http.StatusBadRequest, errorMsg, errorMsg) } } for _, pluginId := range parentPluginIds { if _, ok := pluginParentIdMap[pluginId]; !ok { - nonExistingPluginParentIds = append(nonExistingPluginParentIds, pluginId) + errorMsg := fmt.Sprintf("there are some plugin parent ids in request that do not exist %d", pluginId) + return util.GetApiError(http.StatusBadRequest, errorMsg, errorMsg) } } - if len(nonExistingPluginVersionIds) > 0 { - errorMsg := fmt.Sprintf("there are some plugin version ids in request that do not exist %v", nonExistingPluginVersionIds) - return util.NewApiError().WithHttpStatusCode(http.StatusBadRequest).WithCode(strconv.Itoa(http.StatusBadRequest)).WithUserMessage(errorMsg).WithInternalMessage(errorMsg) - } - if len(nonExistingPluginParentIds) > 0 { - errorMsg := fmt.Sprintf("there are some plugin parent ids in request that do not exist %v", nonExistingPluginParentIds) - return util.NewApiError().WithHttpStatusCode(http.StatusBadRequest).WithCode(strconv.Itoa(http.StatusBadRequest)).WithUserMessage(errorMsg).WithInternalMessage(errorMsg) - } return nil } @@ -1800,7 +1804,7 @@ func (impl *GlobalPluginServiceImpl) GetPluginDetailV2(pluginVersionIds, parentP } pluginParentMetadataDtos := make([]*bean2.PluginParentMetadataDto, 0, len(pluginVersionIds)+len(parentPluginIds)) if len(pluginVersionIds) == 0 && len(parentPluginIds) == 0 { - return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: strconv.Itoa(http.StatusBadRequest), InternalMessage: bean2.NoPluginOrParentIdProvidedErr, UserMessage: bean2.NoPluginOrParentIdProvidedErr} + return nil, util.GetApiError(http.StatusBadRequest, bean2.NoPluginOrParentIdProvidedErr, bean2.NoPluginOrParentIdProvidedErr) } pluginVersionIdsMap, parentPluginIdsMap := helper2.GetPluginVersionAndParentPluginIdsMap(pluginVersionIds, parentPluginIds) @@ -1809,7 +1813,7 @@ func (impl *GlobalPluginServiceImpl) GetPluginDetailV2(pluginVersionIds, parentP filteredPluginVersionMetadata := helper2.GetPluginVersionsMetadataByVersionAndParentPluginIds(pluginVersionsMetadata, pluginVersionIdsMap, parentPluginIdsMap) if len(filteredPluginVersionMetadata) == 0 { - return nil, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: strconv.Itoa(http.StatusNotFound), InternalMessage: bean2.NoPluginFoundForThisSearchQueryErr, UserMessage: bean2.NoPluginFoundForThisSearchQueryErr} + return nil, util.GetApiError(http.StatusNotFound, bean2.NoPluginFoundForThisSearchQueryErr, bean2.NoPluginFoundForThisSearchQueryErr) } for _, version := range filteredPluginVersionMetadata { _, found := pluginVersionIdsMap[version.Id] @@ -1939,6 +1943,7 @@ func (impl *GlobalPluginServiceImpl) GetAllPluginMinData() ([]*bean2.PluginMinDt } pluginMinList := make([]*bean2.PluginMinDto, 0, len(pluginsParentMinData)) for _, item := range pluginsParentMinData { + //since creating new version of preset plugin is disabled for end user, hence ignoring PRESET plugin in min list if item.Type == repository.PLUGIN_TYPE_PRESET { continue } @@ -1955,20 +1960,10 @@ func (impl *GlobalPluginServiceImpl) checkValidationOnPluginNameAndIdentifier(pl } for _, plugin := range plugins { if plugin.Identifier == pluginReq.PluginIdentifier { - return &util.ApiError{ - HttpStatusCode: http.StatusConflict, - Code: strconv.Itoa(http.StatusConflict), - InternalMessage: bean2.PluginWithSameIdentifierExistsError, - UserMessage: bean2.PluginWithSameIdentifierExistsError, - } + return util.GetApiError(http.StatusConflict, bean2.PluginWithSameIdentifierExistsError, bean2.PluginWithSameIdentifierExistsError) } if plugin.Name == pluginReq.Name { - return &util.ApiError{ - HttpStatusCode: http.StatusConflict, - Code: strconv.Itoa(http.StatusConflict), - InternalMessage: bean2.PluginWithSameNameExistError, - UserMessage: bean2.PluginWithSameNameExistError, - } + return util.GetApiError(http.StatusConflict, bean2.PluginWithSameNameExistError, bean2.PluginWithSameNameExistError) } } return nil @@ -1981,26 +1976,20 @@ func (impl *GlobalPluginServiceImpl) checkValidationOnVersion(pluginReq *bean2.P return err } for _, pluginVersion := range pluginVersions { - if pluginVersion.PluginVersion == pluginReq.Versions.DetailedPluginVersionData[0].Version { - return &util.ApiError{ - HttpStatusCode: http.StatusBadRequest, - Code: strconv.Itoa(http.StatusBadRequest), - InternalMessage: bean2.PluginVersionAlreadyExistError, - UserMessage: bean2.PluginVersionAlreadyExistError, + if pluginReq.Versions != nil && len(pluginReq.Versions.DetailedPluginVersionData) > 0 && pluginReq.Versions.DetailedPluginVersionData[0] != nil { + // if plugin version from req is already created then return error + if pluginVersion.PluginVersion == pluginReq.Versions.DetailedPluginVersionData[0].Version { + return util.GetApiError(http.StatusBadRequest, bean2.PluginVersionAlreadyExistError, bean2.PluginVersionAlreadyExistError) } } + } return nil } func (impl *GlobalPluginServiceImpl) validateV2PluginRequest(pluginReq *bean2.PluginParentMetadataDto) error { - if len(pluginReq.Versions.DetailedPluginVersionData) == 0 || pluginReq.Versions.DetailedPluginVersionData[0] == nil { - return &util.ApiError{ - HttpStatusCode: http.StatusBadRequest, - Code: strconv.Itoa(http.StatusBadRequest), - InternalMessage: bean2.NoStepDataToProceedError, - UserMessage: bean2.NoStepDataToProceedError, - } + if pluginReq.Versions == nil || len(pluginReq.Versions.DetailedPluginVersionData) == 0 || pluginReq.Versions.DetailedPluginVersionData[0] == nil { + return util.GetApiError(http.StatusBadRequest, bean2.NoStepDataToProceedError, bean2.NoStepDataToProceedError) } if pluginReq.Id == 0 { //create plugin req. @@ -2016,27 +2005,20 @@ func (impl *GlobalPluginServiceImpl) validateV2PluginRequest(pluginReq *bean2.Pl return err } } - - version := fmt.Sprintf("v%s", pluginReq.Versions.DetailedPluginVersionData[0].Version) + version := pluginReq.Versions.DetailedPluginVersionData[0].Version + if !strings.Contains(version, "v") { + version = fmt.Sprintf("v%s", version) + } // semantic versioning validation on plugin's version if !semver.IsValid(version) { - return &util.ApiError{ - HttpStatusCode: http.StatusBadRequest, - Code: strconv.Itoa(http.StatusBadRequest), - InternalMessage: bean2.PluginVersionNotSemanticallyCorrectError, - UserMessage: bean2.PluginVersionNotSemanticallyCorrectError, - } + return util.GetApiError(http.StatusBadRequest, bean2.PluginVersionNotSemanticallyCorrectError, bean2.PluginVersionNotSemanticallyCorrectError) } //validate icon url and size if len(pluginReq.Icon) > 0 { err := utils.FetchIconAndCheckSize(pluginReq.Icon, bean2.PluginIconMaxSizeInBytes) if err != nil { - return &util.ApiError{ - HttpStatusCode: http.StatusBadRequest, - Code: strconv.Itoa(http.StatusBadRequest), - InternalMessage: fmt.Sprintf("%s err:= %s", bean2.PluginIconNotCorrectOrReachableError, err.Error()), - UserMessage: fmt.Sprintf("%s err:= %s", bean2.PluginIconNotCorrectOrReachableError, err.Error()), - } + errMsg := fmt.Sprintf("%s err:= %s", bean2.PluginIconNotCorrectOrReachableError, err.Error()) + return util.GetApiError(http.StatusBadRequest, errMsg, errMsg) } } return nil @@ -2079,10 +2061,8 @@ func (impl *GlobalPluginServiceImpl) createNewPlugin(tx *pg.Tx, pluginDto *bean2 return 0, err } } else { - return 0, util.NewApiError().WithCode(strconv.Itoa(http.StatusBadRequest)).WithHttpStatusCode(http.StatusBadRequest). - WithInternalMessage(bean2.PluginStepsNotProvidedError).WithUserMessage(errors.New(bean2.PluginStepsNotProvidedError)) + return 0, util.GetApiError(http.StatusBadRequest, bean2.PluginStepsNotProvidedError, bean2.PluginStepsNotProvidedError) } - err = impl.createPluginTagAndRelations(pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) if err != nil { impl.logger.Errorw("createNewPlugin, error in createPluginTagAndRelations", "tags", pluginDto.Versions.DetailedPluginVersionData[0].Tags, "err", err) @@ -2163,8 +2143,7 @@ func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx * return 0, err } } else { - return 0, util.NewApiError().WithCode(strconv.Itoa(http.StatusBadRequest)).WithHttpStatusCode(http.StatusBadRequest). - WithInternalMessage(bean2.PluginStepsNotProvidedError).WithUserMessage(errors.New(bean2.PluginStepsNotProvidedError)) + return 0, util.GetApiError(http.StatusBadRequest, bean2.PluginStepsNotProvidedError, bean2.PluginStepsNotProvidedError) } err = impl.createPluginTagAndRelations(pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) @@ -2179,6 +2158,7 @@ func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx * func (impl *GlobalPluginServiceImpl) CreatePluginOrVersions(pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { err := impl.validateV2PluginRequest(pluginDto) if err != nil { + impl.logger.Errorw("error in validating create plugin request", "pluginReqDto", pluginDto, "err", err) return 0, err } diff --git a/pkg/plugin/bean/bean.go b/pkg/plugin/bean/bean.go index 5f65320a18..c31d346332 100644 --- a/pkg/plugin/bean/bean.go +++ b/pkg/plugin/bean/bean.go @@ -1,7 +1,18 @@ /* * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package bean import ( diff --git a/pkg/plugin/repository/GlobalPluginRepository.go b/pkg/plugin/repository/GlobalPluginRepository.go index 147f6fb725..9cc50748f2 100644 --- a/pkg/plugin/repository/GlobalPluginRepository.go +++ b/pkg/plugin/repository/GlobalPluginRepository.go @@ -1,5 +1,17 @@ /* * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package repository @@ -885,7 +897,7 @@ func (impl *GlobalPluginRepositoryImpl) MarkPreviousPluginVersionLatestFalse(plu var model PluginMetadata _, err := impl.dbConnection.Model(&model). Set("is_latest = ?", false). - Where("id = (select id from plugin_metadata where plugin_parent_metadata_id = ? and is_latest =true order by created_on desc limit 1)", pluginParentId). + Where("id = (select id from plugin_metadata where plugin_parent_metadata_id = ? and is_latest =true order by created_on desc limit ?)", pluginParentId, 1). Update() if err != nil { impl.logger.Errorw("error in updating last version isLatest as false for a plugin parent id", "pluginParentId", pluginParentId, "err", err) From 47015bcba84f2042c7bd68d72701d331b27c50e3 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 9 Aug 2024 11:47:52 +0530 Subject: [PATCH 18/23] wip: code review incorp part -2 --- api/restHandler/GlobalPluginRestHandler.go | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/api/restHandler/GlobalPluginRestHandler.go b/api/restHandler/GlobalPluginRestHandler.go index 21d225e1b6..a599860f17 100644 --- a/api/restHandler/GlobalPluginRestHandler.go +++ b/api/restHandler/GlobalPluginRestHandler.go @@ -429,6 +429,20 @@ func (handler *GlobalPluginRestHandlerImpl) CreatePlugin(w http.ResponseWriter, common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } + token := r.Header.Get("token") + appId, err := common.ExtractIntQueryParam(w, r, "appId", 0) + if err != nil { + return + } + ok, err := handler.IsUserAuthorized(token, appId) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + if !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + return + } decoder := json.NewDecoder(r.Body) var pluginDataDto *bean.PluginParentMetadataDto err = decoder.Decode(pluginDataDto) @@ -437,15 +451,8 @@ func (handler *GlobalPluginRestHandlerImpl) CreatePlugin(w http.ResponseWriter, common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - handler.logger.Infow("request payload received for patching plugins", pluginDataDto, "userId", userId) - // RBAC enforcer applying - token := r.Header.Get("token") - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionUpdate, "*"); !ok { - common.WriteJsonResp(w, errors.New("unauthorized user"), nil, http.StatusForbidden) - return - } + handler.logger.Infow("request payload received for creating plugins", pluginDataDto, "userId", userId) - //RBAC enforcer Ends pluginVersionId, err := handler.globalPluginService.CreatePluginOrVersions(pluginDataDto, userId) if err != nil { handler.logger.Errorw("service error, error in creating plugin", "pluginCreateRequestDto", pluginDataDto, "err", err) From 27e79d50360a8747579e0f4f1f338edadaf9fc6f Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 9 Aug 2024 11:56:28 +0530 Subject: [PATCH 19/23] wip: code review incorp part -3 --- api/restHandler/GlobalPluginRestHandler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/restHandler/GlobalPluginRestHandler.go b/api/restHandler/GlobalPluginRestHandler.go index a599860f17..44a2305c0a 100644 --- a/api/restHandler/GlobalPluginRestHandler.go +++ b/api/restHandler/GlobalPluginRestHandler.go @@ -444,8 +444,8 @@ func (handler *GlobalPluginRestHandlerImpl) CreatePlugin(w http.ResponseWriter, return } decoder := json.NewDecoder(r.Body) - var pluginDataDto *bean.PluginParentMetadataDto - err = decoder.Decode(pluginDataDto) + var pluginDataDto bean.PluginParentMetadataDto + err = decoder.Decode(&pluginDataDto) if err != nil { handler.logger.Errorw("request err, CreatePlugin", "error", err, "payload", pluginDataDto) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) @@ -453,7 +453,7 @@ func (handler *GlobalPluginRestHandlerImpl) CreatePlugin(w http.ResponseWriter, } handler.logger.Infow("request payload received for creating plugins", pluginDataDto, "userId", userId) - pluginVersionId, err := handler.globalPluginService.CreatePluginOrVersions(pluginDataDto, userId) + pluginVersionId, err := handler.globalPluginService.CreatePluginOrVersions(&pluginDataDto, userId) if err != nil { handler.logger.Errorw("service error, error in creating plugin", "pluginCreateRequestDto", pluginDataDto, "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) From 5efc2c5f7939b319fd90a4b5b6432e4c3cd80774 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 9 Aug 2024 12:21:11 +0530 Subject: [PATCH 20/23] wip: remove code duplication --- pkg/plugin/GlobalPluginService.go | 127 +++++++++++++++--------------- 1 file changed, 62 insertions(+), 65 deletions(-) diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index 2a8609f9a7..5423ab22b2 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -2024,53 +2024,6 @@ func (impl *GlobalPluginServiceImpl) validateV2PluginRequest(pluginReq *bean2.Pl return nil } -func (impl *GlobalPluginServiceImpl) createNewPlugin(tx *pg.Tx, pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { - pluginParentMetadata, err := impl.globalPluginRepository.SavePluginParentMetadata(tx, adaptor.GetPluginParentMetadataDbObject(pluginDto, userId)) - if err != nil { - impl.logger.Errorw("createNewPlugin, error in saving plugin parent metadata", "pluginDto", pluginDto, "err", err) - return 0, err - } - pluginDto.Id = pluginParentMetadata.Id - pluginVersionDto := adaptor.GetPluginVersionMetadataDbObject(pluginDto, userId). - WithPluginParentMetadataId(pluginParentMetadata.Id). - WithIsLatestFlag(true) - - pluginVersionMetadata, err := impl.globalPluginRepository.SavePluginMetadata(pluginVersionDto, tx) - if err != nil { - impl.logger.Errorw("createNewPlugin, error in saving plugin version metadata", "pluginDto", pluginDto, "err", err) - return 0, err - } - pluginDto.Versions.DetailedPluginVersionData[0].Id = pluginVersionMetadata.Id - - pluginStageMapping := &repository.PluginStageMapping{ - PluginId: pluginParentMetadata.Id, - StageType: repository.CI_CD, - AuditLog: sql.NewDefaultAuditLog(userId), - } - _, err = impl.globalPluginRepository.SavePluginStageMapping(pluginStageMapping, tx) - if err != nil { - impl.logger.Errorw("createNewPlugin, error in saving plugin stage mapping", "pluginDto", pluginDto, "err", err) - return 0, err - } - - pluginStepsToCreate := pluginDto.Versions.DetailedPluginVersionData[0].PluginSteps - if len(pluginStepsToCreate) > 0 { - err = impl.saveDeepPluginStepData(pluginVersionMetadata.Id, pluginStepsToCreate, userId, tx) - if err != nil { - impl.logger.Errorw("createNewPlugin, error in saving plugin step data", "err", err) - return 0, err - } - } else { - return 0, util.GetApiError(http.StatusBadRequest, bean2.PluginStepsNotProvidedError, bean2.PluginStepsNotProvidedError) - } - err = impl.createPluginTagAndRelations(pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) - if err != nil { - impl.logger.Errorw("createNewPlugin, error in createPluginTagAndRelations", "tags", pluginDto.Versions.DetailedPluginVersionData[0].Tags, "err", err) - return 0, err - } - return pluginVersionMetadata.Id, nil -} - func (impl *GlobalPluginServiceImpl) createPluginTagAndRelations(pluginReq *bean2.PluginsVersionDetail, userId int32, tx *pg.Tx) error { if pluginReq.AreNewTagsPresent { err := impl.CreateNewPluginTagsAndRelationsIfRequiredV2(pluginReq, userId, tx) @@ -2109,6 +2062,62 @@ func (impl *GlobalPluginServiceImpl) CreatePluginTagRelations(pluginReq *bean2.P return nil } +func (impl *GlobalPluginServiceImpl) createPluginStepDataAndTagRelations(pluginVersionId int, pluginVersionDetail *bean2.PluginsVersionDetail, userId int32, tx *pg.Tx) error { + if len(pluginVersionDetail.PluginSteps) > 0 { + err := impl.saveDeepPluginStepData(pluginVersionId, pluginVersionDetail.PluginSteps, userId, tx) + if err != nil { + impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in saving plugin step data", "err", err) + return err + } + } else { + return util.GetApiError(http.StatusBadRequest, bean2.PluginStepsNotProvidedError, bean2.PluginStepsNotProvidedError) + } + + err := impl.createPluginTagAndRelations(pluginVersionDetail, userId, tx) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in createPluginTagAndRelations", "tags", pluginVersionDetail.Tags, "err", err) + return err + } + return nil +} + +func (impl *GlobalPluginServiceImpl) createNewPlugin(tx *pg.Tx, pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { + pluginParentMetadata, err := impl.globalPluginRepository.SavePluginParentMetadata(tx, adaptor.GetPluginParentMetadataDbObject(pluginDto, userId)) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in saving plugin parent metadata", "pluginDto", pluginDto, "err", err) + return 0, err + } + pluginDto.Id = pluginParentMetadata.Id + pluginVersionDto := adaptor.GetPluginVersionMetadataDbObject(pluginDto, userId). + WithPluginParentMetadataId(pluginParentMetadata.Id). + WithIsLatestFlag(true) + + pluginVersionMetadata, err := impl.globalPluginRepository.SavePluginMetadata(pluginVersionDto, tx) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in saving plugin version metadata", "pluginDto", pluginDto, "err", err) + return 0, err + } + pluginDto.Versions.DetailedPluginVersionData[0].Id = pluginVersionMetadata.Id + + pluginStageMapping := &repository.PluginStageMapping{ + PluginId: pluginParentMetadata.Id, + StageType: repository.CI_CD, + AuditLog: sql.NewDefaultAuditLog(userId), + } + _, err = impl.globalPluginRepository.SavePluginStageMapping(pluginStageMapping, tx) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in saving plugin stage mapping", "pluginDto", pluginDto, "err", err) + return 0, err + } + + err = impl.createPluginStepDataAndTagRelations(pluginVersionMetadata.Id, pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) + if err != nil { + impl.logger.Errorw("createNewPlugin, error in createPluginStepDataAndTagRelations", "pluginDto", pluginDto, "err", err) + return 0, err + } + return pluginVersionMetadata.Id, nil +} + func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx *pg.Tx, pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { var pluginParentMinData *repository.PluginParentMetadata var err error @@ -2135,30 +2144,18 @@ func (impl *GlobalPluginServiceImpl) createNewPluginVersionOfExistingPlugin(tx * } pluginDto.Versions.DetailedPluginVersionData[0].Id = pluginVersionMetadata.Id - pluginStepsToCreate := pluginDto.Versions.DetailedPluginVersionData[0].PluginSteps - if len(pluginStepsToCreate) > 0 { - err = impl.saveDeepPluginStepData(pluginVersionMetadata.Id, pluginStepsToCreate, userId, tx) - if err != nil { - impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in saving plugin step data", "err", err) - return 0, err - } - } else { - return 0, util.GetApiError(http.StatusBadRequest, bean2.PluginStepsNotProvidedError, bean2.PluginStepsNotProvidedError) - } - - err = impl.createPluginTagAndRelations(pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) + err = impl.createPluginStepDataAndTagRelations(pluginVersionMetadata.Id, pluginDto.Versions.DetailedPluginVersionData[0], userId, tx) if err != nil { - impl.logger.Errorw("createNewPlugin, error in createPluginTagAndRelations", "tags", pluginDto.Versions.DetailedPluginVersionData[0].Tags, "err", err) + impl.logger.Errorw("createNewPluginVersionOfExistingPlugin, error in createPluginStepDataAndTagRelations", "pluginDto", pluginDto, "err", err) return 0, err } - return pluginVersionMetadata.Id, nil } func (impl *GlobalPluginServiceImpl) CreatePluginOrVersions(pluginDto *bean2.PluginParentMetadataDto, userId int32) (int, error) { err := impl.validateV2PluginRequest(pluginDto) if err != nil { - impl.logger.Errorw("error in validating create plugin request", "pluginReqDto", pluginDto, "err", err) + impl.logger.Errorw("CreatePluginOrVersions, error in validating create plugin request", "pluginReqDto", pluginDto, "err", err) return 0, err } @@ -2174,20 +2171,20 @@ func (impl *GlobalPluginServiceImpl) CreatePluginOrVersions(pluginDto *bean2.Plu // create new version of existing plugin req. versionMetadataId, err = impl.createNewPluginVersionOfExistingPlugin(tx, pluginDto, userId) if err != nil { - impl.logger.Errorw("createPluginV2, error in creating new version of an existing plugin", "existingPluginName", pluginDto.Name, "err", err) + impl.logger.Errorw("CreatePluginOrVersions, error in creating new version of an existing plugin", "existingPluginName", pluginDto.Name, "err", err) return 0, err } } else { // create new plugin req. versionMetadataId, err = impl.createNewPlugin(tx, pluginDto, userId) if err != nil { - impl.logger.Errorw("createPluginV2, error in creating new plugin", "pluginDto", pluginDto, "err", err) + impl.logger.Errorw("CreatePluginOrVersions, error in creating new plugin", "pluginDto", pluginDto, "err", err) return 0, err } } err = tx.Commit() if err != nil { - impl.logger.Errorw("createPluginV2, error in committing db transaction", "err", err) + impl.logger.Errorw("CreatePluginOrVersions, error in committing db transaction", "err", err) return 0, err } return versionMetadataId, nil From d7f077ed9e8cb501914ae1778816a6e69e6f2886 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 9 Aug 2024 15:24:24 +0530 Subject: [PATCH 21/23] wip: hardcode isExposed to true --- pkg/plugin/adaptor/adaptor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/adaptor/adaptor.go b/pkg/plugin/adaptor/adaptor.go index c61dea86ff..e83cdbd966 100644 --- a/pkg/plugin/adaptor/adaptor.go +++ b/pkg/plugin/adaptor/adaptor.go @@ -53,7 +53,7 @@ func GetPluginStepVariableDbObject(pluginStepId int, pluginVariableDto *bean2.Pl Name: pluginVariableDto.Name, Format: pluginVariableDto.Format, Description: pluginVariableDto.Description, - IsExposed: pluginVariableDto.IsExposed, + IsExposed: true, //currently hard coding this, later after plugin creation gets more mature will let user decide AllowEmptyValue: pluginVariableDto.AllowEmptyValue, DefaultValue: pluginVariableDto.DefaultValue, Value: pluginVariableDto.Value, From cbfc11495e8c156ee45347e7514accbb5f18e265 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 9 Aug 2024 17:12:36 +0530 Subject: [PATCH 22/23] wip: hardcode StepType= inline --- pkg/plugin/adaptor/adaptor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/adaptor/adaptor.go b/pkg/plugin/adaptor/adaptor.go index e83cdbd966..49f44ddb0b 100644 --- a/pkg/plugin/adaptor/adaptor.go +++ b/pkg/plugin/adaptor/adaptor.go @@ -22,7 +22,7 @@ func GetPluginStepDbObject(pluginStepDto *bean2.PluginStepsDto, pluginVersionMet Name: pluginStepDto.Name, Description: pluginStepDto.Description, Index: 1, - StepType: pluginStepDto.StepType, + StepType: repository.PLUGIN_STEP_TYPE_INLINE, RefPluginId: pluginStepDto.RefPluginId, OutputDirectoryPath: pluginStepDto.OutputDirectoryPath, DependentOnStep: pluginStepDto.DependentOnStep, From 4060ef5b0383bf69f610363080b3a70d05871dd8 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 12 Aug 2024 09:16:16 +0530 Subject: [PATCH 23/23] wip: set default VariableStepIndex= 1 --- pkg/plugin/adaptor/adaptor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/plugin/adaptor/adaptor.go b/pkg/plugin/adaptor/adaptor.go index 49f44ddb0b..e5e0f50e9d 100644 --- a/pkg/plugin/adaptor/adaptor.go +++ b/pkg/plugin/adaptor/adaptor.go @@ -60,7 +60,7 @@ func GetPluginStepVariableDbObject(pluginStepId int, pluginVariableDto *bean2.Pl VariableType: pluginVariableDto.VariableType, ValueType: pluginVariableDto.ValueType, PreviousStepIndex: pluginVariableDto.PreviousStepIndex, - VariableStepIndex: pluginVariableDto.VariableStepIndex, + VariableStepIndex: 1, //currently hard coding this, later after plugin creation gets more mature will let user decide VariableStepIndexInPlugin: pluginVariableDto.VariableStepIndexInPlugin, ReferenceVariableName: pluginVariableDto.ReferenceVariableName, AuditLog: sql.NewDefaultAuditLog(userId),