feat: native auth kube (#1185)

Signed-off-by: Michal Vala <mvala@redhat.com>

bundle/next-all-namespaces/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml

@@ -126,7 +126,7 @@ metadata:
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
     repository: https://github.com/eclipse-che/che-operator
     support: Eclipse Foundation
-  name: eclipse-che-preview-openshift.v7.40.0-380.next-all-namespaces
+  name: eclipse-che-preview-openshift.v7.40.0-381.next-all-namespaces
   namespace: placeholder
 spec:
   apiservicedefinitions: {}
@@ -1134,6 +1134,10 @@ spec:
                         value: quay.io/openshift/origin-oauth-proxy:4.7
                       - name: RELATED_IMAGE_gateway_authorization_sidecar
                         value: quay.io/openshift/origin-kube-rbac-proxy:4.7
+                      - name: RELATED_IMAGE_gateway_authentication_sidecar_k8s
+                        value: quay.io/oauth2-proxy/oauth2-proxy:v7.2.0
+                      - name: RELATED_IMAGE_gateway_authorization_sidecar_k8s
+                        value: quay.io/brancz/kube-rbac-proxy:v0.11.0
                       - name: RELATED_IMAGE_gateway_header_sidecar
                         value: quay.io/che-incubator/header-rewrite-proxy:latest
                       - name: CHE_FLAVOR
@@ -1439,4 +1443,4 @@ spec:
   maturity: stable
   provider:
     name: Eclipse Foundation
-  version: 7.40.0-380.next-all-namespaces
+  version: 7.40.0-381.next-all-namespaces

bundle/next/eclipse-che-preview-kubernetes/manifests/che-operator.clusterserviceversion.yaml

@@ -133,7 +133,7 @@ metadata:
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
     repository: https://github.com/eclipse-che/che-operator
     support: Eclipse Foundation
-  name: eclipse-che-preview-kubernetes.v7.40.0-380.next
+  name: eclipse-che-preview-kubernetes.v7.40.0-381.next
   namespace: placeholder
 spec:
   apiservicedefinitions: {}
@@ -1123,6 +1123,10 @@ spec:
                         value: quay.io/openshift/origin-oauth-proxy:4.7
                       - name: RELATED_IMAGE_gateway_authorization_sidecar
                         value: quay.io/openshift/origin-kube-rbac-proxy:4.7
+                      - name: RELATED_IMAGE_gateway_authentication_sidecar_k8s
+                        value: quay.io/oauth2-proxy/oauth2-proxy:v7.2.0
+                      - name: RELATED_IMAGE_gateway_authorization_sidecar_k8s
+                        value: quay.io/brancz/kube-rbac-proxy:v0.11.0
                       - name: RELATED_IMAGE_gateway_header_sidecar
                         value: quay.io/che-incubator/header-rewrite-proxy:latest
                       - name: CHE_FLAVOR
@@ -1406,4 +1410,4 @@ spec:
   maturity: stable
   provider:
     name: Eclipse Foundation
-  version: 7.40.0-380.next
+  version: 7.40.0-381.next

bundle/next/eclipse-che-preview-openshift/manifests/che-operator.clusterserviceversion.yaml

@@ -126,7 +126,7 @@ metadata:
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
     repository: https://github.com/eclipse-che/che-operator
     support: Eclipse Foundation
-  name: eclipse-che-preview-openshift.v7.40.0-380.next
+  name: eclipse-che-preview-openshift.v7.40.0-381.next
   namespace: placeholder
 spec:
   apiservicedefinitions: {}
@@ -1134,6 +1134,10 @@ spec:
                         value: quay.io/openshift/origin-oauth-proxy:4.7
                       - name: RELATED_IMAGE_gateway_authorization_sidecar
                         value: quay.io/openshift/origin-kube-rbac-proxy:4.7
+                      - name: RELATED_IMAGE_gateway_authentication_sidecar_k8s
+                        value: quay.io/oauth2-proxy/oauth2-proxy:v7.2.0
+                      - name: RELATED_IMAGE_gateway_authorization_sidecar_k8s
+                        value: quay.io/brancz/kube-rbac-proxy:v0.11.0
                       - name: RELATED_IMAGE_gateway_header_sidecar
                         value: quay.io/che-incubator/header-rewrite-proxy:latest
                       - name: CHE_FLAVOR
@@ -1439,4 +1443,4 @@ spec:
   maturity: stable
   provider:
     name: Eclipse Foundation
-  version: 7.40.0-380.next
+  version: 7.40.0-381.next

config/manager/manager.yaml

@@ -97,6 +97,10 @@ spec:
               value: quay.io/openshift/origin-oauth-proxy:4.7
             - name: RELATED_IMAGE_gateway_authorization_sidecar
               value: quay.io/openshift/origin-kube-rbac-proxy:4.7
+            - name: RELATED_IMAGE_gateway_authentication_sidecar_k8s
+              value: quay.io/oauth2-proxy/oauth2-proxy:v7.2.0
+            - name: RELATED_IMAGE_gateway_authorization_sidecar_k8s
+              value: quay.io/brancz/kube-rbac-proxy:v0.11.0
             - name: RELATED_IMAGE_gateway_header_sidecar
               value: quay.io/che-incubator/header-rewrite-proxy:latest
             - name: CHE_FLAVOR

controllers/che/checluster_controller.go

@@ -523,7 +523,7 @@ func (r *CheClusterReconciler) reconcileFinalizers(deployContext *deploy.DeployC
 		}
 	}
 
-	if util.IsOpenShift && deployContext.CheCluster.IsNativeUserModeEnabled() {
+	if deployContext.CheCluster.IsNativeUserModeEnabled() {
 		if _, err := r.reconcileGatewayPermissionsFinalizers(deployContext); err != nil {
 			logrus.Error(err)
 		}

controllers/che/checluster_controller_test.go

@@ -106,11 +106,11 @@ func TestNativeUserModeEnabled(t *testing.T) {
 			expectedNativeUserValue: pointer.BoolPtr(true),
 		},
 		{
-			name:                    "che-operator should not modify nativeUserMode when not on openshift",
+			name:                    "che-operator should use nativeUserMode when devworkspaces on kubernetes and no initial value in CR for nativeUserMode",
 			isOpenshift:             false,
 			devworkspaceEnabled:     true,
 			initialNativeUserValue:  nil,
-			expectedNativeUserValue: nil,
+			expectedNativeUserValue: pointer.BoolPtr(true),
 		},
 		{
 			name:                    "che-operator not modify nativeUserMode when devworkspace not enabled",

controllers/che/create.go

@@ -140,7 +140,7 @@ func (r *CheClusterReconciler) GenerateAndSaveFields(deployContext *deploy.Deplo
 		}
 	}
 
-	if !util.IsOpenShift || !deployContext.CheCluster.IsNativeUserModeEnabled() {
+	if !deployContext.CheCluster.IsNativeUserModeEnabled() {
 		keycloakRealm := util.GetValue(deployContext.CheCluster.Spec.Auth.IdentityProviderRealm, cheFlavor)
 		if len(deployContext.CheCluster.Spec.Auth.IdentityProviderRealm) < 1 {
 			deployContext.CheCluster.Spec.Auth.IdentityProviderRealm = keycloakRealm
@@ -249,7 +249,7 @@ func (r *CheClusterReconciler) GenerateAndSaveFields(deployContext *deploy.Deplo
 		}
 	}
 
-	if util.IsOpenShift && deployContext.CheCluster.Spec.DevWorkspace.Enable && deployContext.CheCluster.Spec.Auth.NativeUserMode == nil {
+	if deployContext.CheCluster.Spec.DevWorkspace.Enable && deployContext.CheCluster.Spec.Auth.NativeUserMode == nil {
 		newNativeUserModeValue := util.NewBoolPointer(true)
 		deployContext.CheCluster.Spec.Auth.NativeUserMode = newNativeUserModeValue
 		if err := deploy.UpdateCheCRSpec(deployContext, "nativeUserMode", strconv.FormatBool(*newNativeUserModeValue)); err != nil {

controllers/che/gateway_permission.go

@@ -16,7 +16,6 @@ import (
 	orgv1 "github.com/eclipse-che/che-operator/api/v1"
 	"github.com/eclipse-che/che-operator/pkg/deploy"
 	"github.com/eclipse-che/che-operator/pkg/deploy/gateway"
-	"github.com/eclipse-che/che-operator/pkg/util"
 	rbac "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/types"
 )
@@ -26,7 +25,7 @@ const (
 )
 
 func (r *CheClusterReconciler) reconcileGatewayPermissions(deployContext *deploy.DeployContext) (bool, error) {
-	if util.IsOpenShift && deployContext.CheCluster.IsNativeUserModeEnabled() {
+	if deployContext.CheCluster.IsNativeUserModeEnabled() {
 		name := gatewayPermisisonsName(deployContext.CheCluster)
 		if _, err := deploy.SyncClusterRoleToCluster(deployContext, name, getGatewayClusterRoleRules()); err != nil {
 			return false, err

controllers/devworkspace/solver/che_routing.go

@@ -449,13 +449,13 @@ func provisionMainWorkspaceRoute(cheCluster *v2alpha1.CheCluster, routing *dwo.D
 		// on OpenShift, we need to set authorization header.
 		// This MUST come before Auth, because Auth needs Authorization header to be properly set.
 		cfg.AddAuthHeaderRewrite(dwId)
+	}
 
-		// authorize against kube-rbac-proxy in che-gateway. This will be needed for k8s native auth as well.
-		cfg.AddAuth(dwId, "http://127.0.0.1:8089?namespace="+dwNamespace)
+	// authorize against kube-rbac-proxy in che-gateway. This will be needed for k8s native auth as well.
+	cfg.AddAuth(dwId, "http://127.0.0.1:8089?namespace="+dwNamespace)
 
-		// make '/healthz' path of main endpoints reachable from outside
-		routeForHealthzEndpoint(cfg, dwId, routing.Spec.Endpoints)
-	}
+	// make '/healthz' path of main endpoints reachable from outside
+	routeForHealthzEndpoint(cfg, dwId, routing.Spec.Endpoints)
 
 	if contents, err := yaml.Marshal(cfg); err != nil {
 		return nil, err
@@ -514,13 +514,10 @@ func addEndpointToTraefikConfig(componentName string, e dw.Endpoint, cfg *gatewa
 		100,
 		fmt.Sprintf("http://127.0.0.1:%d", e.TargetPort),
 		[]string{prefix})
-	if util.IsOpenShift4 {
-		cfg.AddAuth(name, fmt.Sprintf("http://%s.%s:8089?namespace=%s", gateway.GatewayServiceName, cheCluster.Namespace, routing.Namespace))
-	}
+	cfg.AddAuth(name, fmt.Sprintf("http://%s.%s:8089?namespace=%s", gateway.GatewayServiceName, cheCluster.Namespace, routing.Namespace))
 
 	// we need to disable auth for '/healthz' path in main endpoint, for now only on OpenShift
-	if util.IsOpenShift4 &&
-		e.Attributes.GetString(string(dwo.TypeEndpointAttribute), nil) == string(dwo.MainEndpointType) {
+	if e.Attributes.GetString(string(dwo.TypeEndpointAttribute), nil) == string(dwo.MainEndpointType) {
 		healthzName := name + "-healthz"
 		healthzPath := prefix + "/healthz"
 		cfg.AddComponent(

controllers/devworkspace/solver/che_routing_test.go

@@ -267,9 +267,7 @@ func TestCreateRelocatedObjectsK8S(t *testing.T) {
 		cms := &corev1.ConfigMapList{}
 		cl.List(context.TODO(), cms)
 
-		if len(cms.Items) != 2 {
-			t.Errorf("there should be 2 configmaps created for the gateway config of the workspace but there were: %d", len(cms.Items))
-		}
+		assert.Len(t, cms.Items, 2)
 
 		var workspaceMainCfg *corev1.ConfigMap
 		var workspaceCfg *corev1.ConfigMap
@@ -282,59 +280,74 @@ func TestCreateRelocatedObjectsK8S(t *testing.T) {
 			}
 		}
 
-		if workspaceMainCfg == nil {
-			t.Fatalf("traefik configuration for the workspace not found")
-		}
+		assert.NotNil(t, workspaceMainCfg)
 
 		traefikMainWorkspaceConfig := workspaceMainCfg.Data["wsid.yml"]
-
-		if len(traefikMainWorkspaceConfig) == 0 {
-			t.Fatal("No traefik config file found in the main workspace config configmap")
-		}
+		assert.NotEmpty(t, traefikMainWorkspaceConfig)
 
 		traefikWorkspaceConfig := workspaceCfg.Data["workspace.yml"]
-		if len(traefikWorkspaceConfig) == 0 {
-			t.Fatal("No traefik config file found in the workspace config configmap")
-		}
+		assert.NotEmpty(t, traefikWorkspaceConfig)
 
 		workspaceConfig := gateway.TraefikConfig{}
-		if err := yaml.Unmarshal([]byte(traefikWorkspaceConfig), &workspaceConfig); err != nil {
-			t.Fatal(err)
-		}
-
-		if len(workspaceConfig.HTTP.Routers) != 1 {
-			t.Fatalf("Expected 1 traefik router but got %d", len(workspaceConfig.HTTP.Routers))
-		}
+		assert.NoError(t, yaml.Unmarshal([]byte(traefikWorkspaceConfig), &workspaceConfig))
+		assert.Len(t, workspaceConfig.HTTP.Routers, 2)
 
 		wsid := "wsid-m1-9999"
-		if _, ok := workspaceConfig.HTTP.Routers[wsid]; !ok {
-			t.Fatal("traefik config doesn't contain expected workspace configuration")
-		}
-
-		if len(workspaceConfig.HTTP.Routers[wsid].Middlewares) != 1 {
-			t.Fatalf("Expected 1 middlewares in router but got '%d'", len(workspaceConfig.HTTP.Routers[wsid].Middlewares))
-		}
-
-		if len(workspaceConfig.HTTP.Middlewares) != 1 {
-			t.Fatalf("Expected 1 middlewares set but got '%d'", len(workspaceConfig.HTTP.Middlewares))
-		}
+		assert.Contains(t, workspaceConfig.HTTP.Routers, wsid)
+		assert.Len(t, workspaceConfig.HTTP.Routers[wsid].Middlewares, 2)
+		assert.Len(t, workspaceConfig.HTTP.Middlewares, 3)
 
 		mwares := []string{wsid + gateway.StripPrefixMiddlewareSuffix}
 		for _, mware := range mwares {
-			if _, ok := workspaceConfig.HTTP.Middlewares[mware]; !ok {
-				t.Fatalf("traefik config doesn't set middleware '%s'", mware)
-			}
+			assert.Contains(t, workspaceConfig.HTTP.Middlewares, mware)
 			found := false
 			for _, r := range workspaceConfig.HTTP.Routers[wsid].Middlewares {
 				if r == mware {
 					found = true
 				}
 			}
-			if !found {
-				t.Fatalf("traefik config route doesn't set middleware '%s'", mware)
+			assert.True(t, found)
+		}
+
+		workspaceMainConfig := gateway.TraefikConfig{}
+		assert.NoError(t, yaml.Unmarshal([]byte(traefikMainWorkspaceConfig), &workspaceMainConfig))
+		assert.Len(t, workspaceMainConfig.HTTP.Middlewares, 2)
+
+		wsid = "wsid"
+		mwares = []string{
+			wsid + gateway.AuthMiddlewareSuffix,
+			wsid + gateway.StripPrefixMiddlewareSuffix}
+		for _, mware := range mwares {
+			assert.Contains(t, workspaceMainConfig.HTTP.Middlewares, mware)
+
+			found := false
+			for _, r := range workspaceMainConfig.HTTP.Routers[wsid].Middlewares {
+				if r == mware {
+					found = true
+				}
 			}
+			assert.Truef(t, found, "traefik config route doesn't set middleware '%s'", mware)
 		}
 
+		t.Run("testHealthzEndpointInMainWorkspaceRoute", func(t *testing.T) {
+			healthzName := "9999-healthz"
+			assert.Contains(t, workspaceMainConfig.HTTP.Routers, healthzName)
+			assert.Equal(t, workspaceMainConfig.HTTP.Routers[healthzName].Service, wsid)
+			assert.Equal(t, workspaceMainConfig.HTTP.Routers[healthzName].Rule, "Path(`/wsid/m1/9999/healthz`)")
+			assert.NotContains(t, workspaceMainConfig.HTTP.Routers[healthzName].Middlewares, "wsid"+gateway.AuthMiddlewareSuffix)
+			assert.Contains(t, workspaceMainConfig.HTTP.Routers[healthzName].Middlewares, "wsid"+gateway.StripPrefixMiddlewareSuffix)
+			assert.NotContains(t, workspaceMainConfig.HTTP.Routers[healthzName].Middlewares, "wsid"+gateway.HeaderRewriteMiddlewareSuffix)
+		})
+
+		t.Run("testHealthzEndpointInWorkspaceRoute", func(t *testing.T) {
+			healthzName := "wsid-m1-9999-healthz"
+			assert.Contains(t, workspaceConfig.HTTP.Routers, healthzName)
+			assert.Equal(t, workspaceConfig.HTTP.Routers[healthzName].Service, healthzName)
+			assert.Equal(t, workspaceConfig.HTTP.Routers[healthzName].Rule, "Path(`/m1/9999/healthz`)")
+			assert.NotContains(t, workspaceConfig.HTTP.Routers[healthzName].Middlewares, healthzName+gateway.AuthMiddlewareSuffix)
+			assert.Contains(t, workspaceConfig.HTTP.Routers[healthzName].Middlewares, healthzName+gateway.StripPrefixMiddlewareSuffix)
+		})
+
 	})
 }
 

helmcharts/next/templates/manager.yaml

@@ -97,6 +97,10 @@ spec:
               value: quay.io/openshift/origin-oauth-proxy:4.7
             - name: RELATED_IMAGE_gateway_authorization_sidecar
               value: quay.io/openshift/origin-kube-rbac-proxy:4.7
+            - name: RELATED_IMAGE_gateway_authentication_sidecar_k8s
+              value: quay.io/oauth2-proxy/oauth2-proxy:v7.2.0
+            - name: RELATED_IMAGE_gateway_authorization_sidecar_k8s
+              value: quay.io/brancz/kube-rbac-proxy:v0.11.0
             - name: RELATED_IMAGE_gateway_header_sidecar
               value: quay.io/che-incubator/header-rewrite-proxy:latest
             - name: CHE_FLAVOR

pkg/deploy/defaults.go

@@ -211,6 +211,8 @@ func InitDefaultsFromFile(defaultsPath string) {
 	// Don't get some k8s specific env
 	if !util.IsOpenShift {
 		defaultCheTLSSecretsCreationJobImage = util.GetDeploymentEnv(operatorDeployment, util.GetArchitectureDependentEnv("RELATED_IMAGE_che_tls_secrets_creation_job"))
+		defaultGatewayAuthenticationSidecarImage = util.GetDeploymentEnv(operatorDeployment, util.GetArchitectureDependentEnv("RELATED_IMAGE_gateway_authentication_sidecar_k8s"))
+		defaultGatewayAuthorizationSidecarImage = util.GetDeploymentEnv(operatorDeployment, util.GetArchitectureDependentEnv("RELATED_IMAGE_gateway_authorization_sidecar_k8s"))
 	}
 }
 
@@ -494,6 +496,8 @@ func InitDefaultsFromEnv() {
 	// Don't get some k8s specific env
 	if !util.IsOpenShift {
 		defaultCheTLSSecretsCreationJobImage = getDefaultFromEnv(util.GetArchitectureDependentEnv("RELATED_IMAGE_che_tls_secrets_creation_job"))
+		defaultGatewayAuthenticationSidecarImage = getDefaultFromEnv(util.GetArchitectureDependentEnv("RELATED_IMAGE_gateway_authentication_sidecar_k8s"))
+		defaultGatewayAuthorizationSidecarImage = getDefaultFromEnv(util.GetArchitectureDependentEnv("RELATED_IMAGE_gateway_authorization_sidecar_k8s"))
 	}
 }
 

pkg/deploy/dev-workspace/dev_workspace_test.go

@@ -86,6 +86,7 @@ func TestReconcileDevWorkspace(t *testing.T) {
 					},
 					Server: orgv1.CheClusterSpecServer{
 						ServerExposureStrategy: "multi-host",
+						CustomCheProperties:    map[string]string{"CHE_INFRA_KUBERNETES_ENABLE__UNSUPPORTED__K8S": "true"},
 					},
 					K8s: orgv1.CheClusterSpecK8SOnly{
 						IngressDomain: "che.domain",
@@ -110,6 +111,7 @@ func TestReconcileDevWorkspace(t *testing.T) {
 					},
 					Server: orgv1.CheClusterSpecServer{
 						ServerExposureStrategy: "single-host",
+						CustomCheProperties:    map[string]string{"CHE_INFRA_KUBERNETES_ENABLE__UNSUPPORTED__K8S": "true"},
 					},
 					K8s: orgv1.CheClusterSpecK8SOnly{
 						IngressDomain: "che.domain",