diff --git a/controllers/csm_controller.go b/controllers/csm_controller.go index 57f798b7e..213376763 100644 --- a/controllers/csm_controller.go +++ b/controllers/csm_controller.go @@ -370,7 +370,7 @@ func (r *ContainerStorageModuleReconciler) handleDeploymentUpdate(oldObj interfa return } - log.Debugw("deployment modified generation", d.Generation, old.Generation) + log.Debugw("deployment modified generation", d.Name, d.Generation, old.Generation) desired := d.Status.Replicas available := d.Status.AvailableReplicas @@ -687,7 +687,8 @@ func (r *ContainerStorageModuleReconciler) SyncCSM(ctx context.Context, cr csmv1 log := logger.GetLogger(ctx) // Create/Update Authorization Proxy Server - if authorizationEnabled, _ := utils.IsModuleEnabled(ctx, cr, csmv1.AuthorizationServer); authorizationEnabled { + authorizationEnabled, _ := utils.IsModuleEnabled(ctx, cr, csmv1.AuthorizationServer) + if authorizationEnabled { log.Infow("Create/Update authorization") if err := r.reconcileAuthorization(ctx, false, operatorConfig, cr, ctrlClient); err != nil { return fmt.Errorf("failed to deploy authorization proxy server: %v", err) @@ -851,9 +852,11 @@ func (r *ContainerStorageModuleReconciler) SyncCSM(ctx context.Context, cr csmv1 return err } - // Create/Update DeamonSet - if err = daemonset.SyncDaemonset(ctx, node.DaemonSetApplyConfig, cluster.ClusterK8sClient, cr.Name); err != nil { - return err + // Create/Update DeamonSet, except for auth proxy + if !authorizationEnabled { + if err = daemonset.SyncDaemonset(ctx, node.DaemonSetApplyConfig, cluster.ClusterK8sClient, cr.Name); err != nil { + return err + } } if replicationEnabled { @@ -1204,7 +1207,7 @@ func (r *ContainerStorageModuleReconciler) removeDriver(ctx context.Context, ins return nil } -// removeModule - remove authorization proxy server +// removeModule - remove standalone modules func (r *ContainerStorageModuleReconciler) removeModule(ctx context.Context, instance csmv1.ContainerStorageModule, operatorConfig utils.OperatorConfig, ctrlClient client.Client) error { log := logger.GetLogger(ctx) diff --git a/operatorconfig/moduleconfig/authorization/v1.7.0/deployment.yaml b/operatorconfig/moduleconfig/authorization/v1.7.0/deployment.yaml index 935fdbc80..25051dd6d 100644 --- a/operatorconfig/moduleconfig/authorization/v1.7.0/deployment.yaml +++ b/operatorconfig/moduleconfig/authorization/v1.7.0/deployment.yaml @@ -14,6 +14,7 @@ spec: template: metadata: labels: + csm: app: proxy-server spec: containers: @@ -92,6 +93,7 @@ spec: template: metadata: labels: + csm: app: tenant-service spec: containers: @@ -176,6 +178,7 @@ spec: template: metadata: labels: + csm: app: role-service spec: serviceAccountName: role-service @@ -254,6 +257,7 @@ spec: template: metadata: labels: + csm: app: storage-service spec: serviceAccountName: storage-service @@ -316,6 +320,7 @@ spec: template: metadata: labels: + csm: app: redis role: primary tier: backend @@ -367,6 +372,7 @@ spec: template: metadata: labels: + csm: app: redis-commander tier: backend spec: @@ -496,4 +502,4 @@ roleRef: subjects: - kind: Group name: system:serviceaccounts:authorization - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io diff --git a/operatorconfig/moduleconfig/authorization/v1.7.0/nginx-ingress-controller.yaml b/operatorconfig/moduleconfig/authorization/v1.7.0/nginx-ingress-controller.yaml index 3bafbb56f..bd6feeab0 100644 --- a/operatorconfig/moduleconfig/authorization/v1.7.0/nginx-ingress-controller.yaml +++ b/operatorconfig/moduleconfig/authorization/v1.7.0/nginx-ingress-controller.yaml @@ -426,6 +426,7 @@ spec: template: metadata: labels: + csm: app.kubernetes.io/component: controller app.kubernetes.io/instance: app.kubernetes.io/name: ingress-nginx @@ -660,4 +661,4 @@ webhooks: resources: - ingresses sideEffects: None - \ No newline at end of file + diff --git a/operatorconfig/moduleconfig/authorization/v1.8.0/deployment.yaml b/operatorconfig/moduleconfig/authorization/v1.8.0/deployment.yaml index 935fdbc80..25051dd6d 100644 --- a/operatorconfig/moduleconfig/authorization/v1.8.0/deployment.yaml +++ b/operatorconfig/moduleconfig/authorization/v1.8.0/deployment.yaml @@ -14,6 +14,7 @@ spec: template: metadata: labels: + csm: app: proxy-server spec: containers: @@ -92,6 +93,7 @@ spec: template: metadata: labels: + csm: app: tenant-service spec: containers: @@ -176,6 +178,7 @@ spec: template: metadata: labels: + csm: app: role-service spec: serviceAccountName: role-service @@ -254,6 +257,7 @@ spec: template: metadata: labels: + csm: app: storage-service spec: serviceAccountName: storage-service @@ -316,6 +320,7 @@ spec: template: metadata: labels: + csm: app: redis role: primary tier: backend @@ -367,6 +372,7 @@ spec: template: metadata: labels: + csm: app: redis-commander tier: backend spec: @@ -496,4 +502,4 @@ roleRef: subjects: - kind: Group name: system:serviceaccounts:authorization - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io diff --git a/operatorconfig/moduleconfig/authorization/v1.8.0/nginx-ingress-controller.yaml b/operatorconfig/moduleconfig/authorization/v1.8.0/nginx-ingress-controller.yaml index 3bafbb56f..bd6feeab0 100644 --- a/operatorconfig/moduleconfig/authorization/v1.8.0/nginx-ingress-controller.yaml +++ b/operatorconfig/moduleconfig/authorization/v1.8.0/nginx-ingress-controller.yaml @@ -426,6 +426,7 @@ spec: template: metadata: labels: + csm: app.kubernetes.io/component: controller app.kubernetes.io/instance: app.kubernetes.io/name: ingress-nginx @@ -660,4 +661,4 @@ webhooks: resources: - ingresses sideEffects: None - \ No newline at end of file + diff --git a/operatorconfig/moduleconfig/authorization/v1.9.0/deployment.yaml b/operatorconfig/moduleconfig/authorization/v1.9.0/deployment.yaml index 935fdbc80..c171dd8e6 100644 --- a/operatorconfig/moduleconfig/authorization/v1.9.0/deployment.yaml +++ b/operatorconfig/moduleconfig/authorization/v1.9.0/deployment.yaml @@ -15,6 +15,7 @@ spec: metadata: labels: app: proxy-server + csm: spec: containers: - name: proxy-server @@ -93,6 +94,7 @@ spec: metadata: labels: app: tenant-service + csm: spec: containers: - name: tenant-service @@ -176,6 +178,7 @@ spec: template: metadata: labels: + csm: app: role-service spec: serviceAccountName: role-service @@ -254,6 +257,7 @@ spec: template: metadata: labels: + csm: app: storage-service spec: serviceAccountName: storage-service @@ -316,6 +320,7 @@ spec: template: metadata: labels: + csm: app: redis role: primary tier: backend @@ -367,6 +372,7 @@ spec: template: metadata: labels: + csm: app: redis-commander tier: backend spec: @@ -496,4 +502,4 @@ roleRef: subjects: - kind: Group name: system:serviceaccounts:authorization - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io diff --git a/operatorconfig/moduleconfig/authorization/v1.9.0/nginx-ingress-controller.yaml b/operatorconfig/moduleconfig/authorization/v1.9.0/nginx-ingress-controller.yaml index 3bafbb56f..bd6feeab0 100644 --- a/operatorconfig/moduleconfig/authorization/v1.9.0/nginx-ingress-controller.yaml +++ b/operatorconfig/moduleconfig/authorization/v1.9.0/nginx-ingress-controller.yaml @@ -426,6 +426,7 @@ spec: template: metadata: labels: + csm: app.kubernetes.io/component: controller app.kubernetes.io/instance: app.kubernetes.io/name: ingress-nginx @@ -660,4 +661,4 @@ webhooks: resources: - ingresses sideEffects: None - \ No newline at end of file + diff --git a/operatorconfig/moduleconfig/authorization/v1.9.1/deployment.yaml b/operatorconfig/moduleconfig/authorization/v1.9.1/deployment.yaml index 935fdbc80..22ce47e52 100644 --- a/operatorconfig/moduleconfig/authorization/v1.9.1/deployment.yaml +++ b/operatorconfig/moduleconfig/authorization/v1.9.1/deployment.yaml @@ -15,6 +15,7 @@ spec: metadata: labels: app: proxy-server + csm: spec: containers: - name: proxy-server @@ -93,6 +94,7 @@ spec: metadata: labels: app: tenant-service + csm: spec: containers: - name: tenant-service @@ -176,6 +178,7 @@ spec: template: metadata: labels: + csm: app: role-service spec: serviceAccountName: role-service @@ -255,6 +258,7 @@ spec: metadata: labels: app: storage-service + csm: spec: serviceAccountName: storage-service containers: @@ -319,6 +323,7 @@ spec: app: redis role: primary tier: backend + csm: spec: containers: - name: primary @@ -369,6 +374,7 @@ spec: labels: app: redis-commander tier: backend + csm: spec: containers: - name: redis-commander @@ -496,4 +502,4 @@ roleRef: subjects: - kind: Group name: system:serviceaccounts:authorization - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io diff --git a/operatorconfig/moduleconfig/authorization/v1.9.1/nginx-ingress-controller.yaml b/operatorconfig/moduleconfig/authorization/v1.9.1/nginx-ingress-controller.yaml index 3bafbb56f..135f8afa5 100644 --- a/operatorconfig/moduleconfig/authorization/v1.9.1/nginx-ingress-controller.yaml +++ b/operatorconfig/moduleconfig/authorization/v1.9.1/nginx-ingress-controller.yaml @@ -429,6 +429,7 @@ spec: app.kubernetes.io/component: controller app.kubernetes.io/instance: app.kubernetes.io/name: ingress-nginx + csm: spec: containers: - args: @@ -660,4 +661,4 @@ webhooks: resources: - ingresses sideEffects: None - \ No newline at end of file + diff --git a/pkg/modules/application_mobility.go b/pkg/modules/application_mobility.go index 9248a48e9..94df5dd53 100644 --- a/pkg/modules/application_mobility.go +++ b/pkg/modules/application_mobility.go @@ -105,8 +105,6 @@ const ( AppMobCertManagerComponent = "cert-manager" // AppMobVeleroComponent - velero component AppMobVeleroComponent = "velero" - // CSMName - name - CSMName = "" ) // getAppMobilityModule - get instance of app mobility module diff --git a/pkg/modules/authorization.go b/pkg/modules/authorization.go index f9360410e..f49faf5b7 100644 --- a/pkg/modules/authorization.go +++ b/pkg/modules/authorization.go @@ -493,6 +493,7 @@ func getAuthorizationServerDeployment(op utils.OperatorConfig, cr csmv1.Containe YamlString = strings.ReplaceAll(YamlString, AuthStorageServiceImage, component.StorageService) YamlString = strings.ReplaceAll(YamlString, AuthRedisImage, component.Redis) YamlString = strings.ReplaceAll(YamlString, AuthRedisCommanderImage, component.Commander) + YamlString = strings.ReplaceAll(YamlString, CSMName, cr.Name) for _, env := range component.Envs { if env.Name == "REDIS_STORAGE_CLASS" { @@ -504,6 +505,7 @@ func getAuthorizationServerDeployment(op utils.OperatorConfig, cr csmv1.Containe YamlString = strings.ReplaceAll(YamlString, AuthNamespace, authNamespace) YamlString = strings.ReplaceAll(YamlString, AuthRedisStorageClass, redisStorageClass) + YamlString = strings.ReplaceAll(YamlString, CSMName, cr.Name) return YamlString, nil } @@ -570,6 +572,7 @@ func getAuthorizationIngressRules(op utils.OperatorConfig, cr csmv1.ContainerSto YamlString = strings.ReplaceAll(YamlString, AuthProxyHost, authHostname) YamlString = strings.ReplaceAll(YamlString, AuthProxyIngressHost, proxyIngressHost) YamlString = strings.ReplaceAll(YamlString, AuthProxyIngressClassName, proxyIngressClassName) + YamlString = strings.ReplaceAll(YamlString, CSMName, cr.Name) return YamlString, nil } @@ -625,6 +628,7 @@ func getNginxIngressController(op utils.OperatorConfig, cr csmv1.ContainerStorag YamlString = string(buf) authNamespace := cr.Namespace YamlString = strings.ReplaceAll(YamlString, AuthNamespace, authNamespace) + YamlString = strings.ReplaceAll(YamlString, CSMName, cr.Name) return YamlString, nil } diff --git a/pkg/modules/commonconfig.go b/pkg/modules/commonconfig.go index 7deb10719..000a325fa 100644 --- a/pkg/modules/commonconfig.go +++ b/pkg/modules/commonconfig.go @@ -33,6 +33,8 @@ const ( CertManagerManifest = "cert-manager.yaml" // CommonNamespace - CommonNamespace = "" + // CSMName - name + CSMName = "" ) // SupportedDriverParam - diff --git a/pkg/utils/status.go b/pkg/utils/status.go index 9c2353300..c540005b6 100644 --- a/pkg/utils/status.go +++ b/pkg/utils/status.go @@ -41,6 +41,7 @@ var dMutex sync.RWMutex var checkModuleStatus = map[csmv1.ModuleType]func(context.Context, *csmv1.ContainerStorageModule, ReconcileCSM, *csmv1.ContainerStorageModuleStatus) (bool, error){ csmv1.Observability: observabilityStatusCheck, csmv1.ApplicationMobility: appMobStatusCheck, + csmv1.AuthorizationServer: authProxyStatusCheck, } func getInt32(pointer *int32) int32 { @@ -389,6 +390,8 @@ func calculateState(ctx context.Context, instance *csmv1.ContainerStorageModule, log := logger.GetLogger(ctx) running := true var err error = nil + nodeStatusGood := true + newStatus.State = constants.Succeeded // TODO: Currently commented this block of code as the API used to get the latest deployment status is not working as expected // TODO: Can be uncommented once this issues gets sorted out controllerReplicas, controllerStatus, controllerErr := getDeploymentStatus(ctx, instance, r) @@ -396,18 +399,28 @@ func calculateState(ctx context.Context, instance *csmv1.ContainerStorageModule, log.Infof("error from getDeploymentStatus: %s", controllerErr.Error()) } + // Auth proxy has no daemonset. Putting this if/else in here and setting nodeStatusGood to true by + // default is a little hacky but will be fixed when we refactor the status code in CSM 1.10 or 1.11 + log.Infof("instance.GetName() is %s", instance.GetName()) + if instance.GetName() != string(csmv1.Authorization) { + expected, nodeStatus, daemonSetErr := getDaemonSetStatus(ctx, instance, r) + newStatus.NodeStatus = nodeStatus + if daemonSetErr != nil { + err = daemonSetErr + log.Infof("calculate Daemonseterror msg [%s]", daemonSetErr.Error()) + } + + log.Infof("daemonset expected [%d]", expected) + log.Infof("daemonset nodeStatus.Available [%s]", nodeStatus.Available) + nodeStatusGood = (fmt.Sprintf("%d", expected) == nodeStatus.Available) + } + newStatus.ControllerStatus = controllerStatus - expected, nodeStatus, daemonSetErr := getDaemonSetStatus(ctx, instance, r) - newStatus.NodeStatus = nodeStatus - newStatus.State = constants.Succeeded log.Infof("deployment controllerReplicas [%s]", controllerReplicas) log.Infof("deployment controllerStatus.Available [%s]", controllerStatus.Available) - log.Infof("daemonset expected [%d]", expected) - log.Infof("daemonset nodeStatus.Available [%s]", nodeStatus.Available) - - if (fmt.Sprintf("%d", controllerReplicas) == controllerStatus.Available) && (fmt.Sprintf("%d", expected) == nodeStatus.Available) { + if (fmt.Sprintf("%d", controllerReplicas) == controllerStatus.Available) && nodeStatusGood { for _, module := range instance.Spec.Modules { moduleStatus, exists := checkModuleStatus[module.Name] @@ -420,21 +433,22 @@ func calculateState(ctx context.Context, instance *csmv1.ContainerStorageModule, if !moduleRunning { running = false newStatus.State = constants.Failed - log.Infof("%s module not running", module) + log.Infof("%s module not running", module.Name) break } + log.Infof("%s module running", module.Name) } } } else { + log.Infof("either controllerReplicas != controllerStatus.Available or nodeStatus is bad") + log.Infof("controllerReplicas", controllerReplicas) + log.Infof("controllerStatus.Available", controllerStatus.Available) + log.Infof("nodeStatusGood", nodeStatusGood) running = false newStatus.State = constants.Failed } - if daemonSetErr != nil { - err = daemonSetErr - log.Infof("calculate Daemonseterror msg [%s]", daemonSetErr.Error()) - } - + log.Infof("setting status to ", "newStatus", newStatus) SetStatus(ctx, r, instance, newStatus) return running, err } @@ -607,18 +621,21 @@ func HandleSuccess(ctx context.Context, instance *csmv1.ContainerStorageModule, log := logger.GetLogger(ctx) running, err := calculateState(ctx, instance, r, newStatus) + log.Info("calculateState returns ", "running", running) if err != nil { log.Error("HandleSuccess Driver status ", "error", err.Error()) newStatus.State = constants.Failed } if running { - newStatus.State = constants.Running + newStatus.State = constants.Succeeded } log.Infow("HandleSuccess Driver state ", "newStatus.State", newStatus.State) - if newStatus.State == constants.Running { + if newStatus.State == constants.Succeeded { // If previously we were in running state - if oldStatus.State == constants.Running { - log.Info("HandleSuccess Driver state didn't change from Running") + if oldStatus.State == constants.Succeeded { + log.Info("HandleSuccess Driver state didn't change from Succeeded") + } else { + log.Info("HandleSuccess Driver stat changed to Succeeded") } return reconcile.Result{}, nil } @@ -858,9 +875,8 @@ func observabilityStatusCheck(ctx context.Context, instance *csmv1.ContainerStor } } - namespace := "karavi" opts := []client.ListOption{ - client.InNamespace(namespace), + client.InNamespace(ObservabilityNamespace), } deploymentList := &appsv1.DeploymentList{} err := r.GetClient().List(ctx, deploymentList, opts...) @@ -879,11 +895,11 @@ func observabilityStatusCheck(ctx context.Context, instance *csmv1.ContainerStor if otelEnabled { otelRunning = checkFn(&deployment) } - case fmt.Sprintf("%s-metrics-%s", namespace, instance.Spec.Driver.CSIDriverType): + case fmt.Sprintf("%s-metrics-%s", ObservabilityNamespace, instance.Spec.Driver.CSIDriverType): if metricsEnabled { metricsRunning = checkFn(&deployment) } - case fmt.Sprintf("%s-topology", namespace): + case fmt.Sprintf("%s-topology", ObservabilityNamespace): if topologyEnabled { topologyRunning = checkFn(&deployment) } @@ -945,3 +961,104 @@ func observabilityStatusCheck(ctx context.Context, instance *csmv1.ContainerStor return false, nil } + +// authProxyStatusCheck - calculate success state for auth proxy +func authProxyStatusCheck(ctx context.Context, instance *csmv1.ContainerStorageModule, r ReconcileCSM, _ *csmv1.ContainerStorageModuleStatus) (bool, error) { + log := logger.GetLogger(ctx) + certEnabled := false + nginxEnabled := false + + for _, m := range instance.Spec.Modules { + if m.Name == csmv1.AuthorizationServer { + for _, c := range m.Components { + if c.Name == "ingress-nginx" && *c.Enabled { + nginxEnabled = true + } + if c.Name == "cert-manager" && *c.Enabled { + certEnabled = true + } + } + } + } + + opts := []client.ListOption{ + client.InNamespace(instance.GetNamespace()), + } + deploymentList := &appsv1.DeploymentList{} + err := r.GetClient().List(ctx, deploymentList, opts...) + if err != nil { + return false, err + } + + checkFn := func(deployment *appsv1.Deployment) bool { + return deployment.Status.ReadyReplicas == *deployment.Spec.Replicas + } + + for _, deployment := range deploymentList.Items { + deployment := deployment + switch deployment.Name { + case "authorization-ingress-nginx-controller": + if nginxEnabled { + if !checkFn(&deployment) { + log.Info("%s component not running in auth proxy deployment", deployment.Name) + return false, nil + } + } + case "cert-manager": + if certEnabled { + if !checkFn(&deployment) { + log.Info("%s component not running in auth proxy deployment", deployment.Name) + return false, nil + } + } + case "cert-manager-cainjector": + if certEnabled { + if !checkFn(&deployment) { + log.Info("%s component not running in auth proxy deployment", deployment.Name) + return false, nil + } + } + case "cert-manager-webhook": + if certEnabled { + if !checkFn(&deployment) { + log.Info("%s component not running in auth proxy deployment", deployment.Name) + return false, nil + } + } + case "proxy-server": + if !checkFn(&deployment) { + log.Info("%s component not running in auth proxy deployment", deployment.Name) + return false, nil + } + case "redis-commander": + if !checkFn(&deployment) { + log.Info("%s component not running in auth proxy deployment", deployment.Name) + return false, nil + } + case "redis-primary": + if !checkFn(&deployment) { + log.Info("%s component not running in auth proxy deployment", deployment.Name) + return false, nil + } + case "role-service": + if !checkFn(&deployment) { + log.Info("%s component not running in auth proxy deployment", deployment.Name) + return false, nil + } + case "storage-service": + if !checkFn(&deployment) { + log.Info("%s component not running in auth proxy deployment", deployment.Name) + return false, nil + } + case "tenant-service": + if !checkFn(&deployment) { + log.Info("%s component not running in auth proxy deployment", deployment.Name) + return false, nil + } + } + } + + log.Info("auth proxy deployment successful") + + return true, nil +}