Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: accessible workspace healthz endpoint #1119

Merged
merged 8 commits into from
Oct 11, 2021
Merged

feat: accessible workspace healthz endpoint #1119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
bcdba60
feat: make /healthz of main workspace endpoint reachable from outside
sparkoo Oct 1, 2021
2166a6f
refactor traefik configurations to more utilize util methods
sparkoo Oct 1, 2021
e0f3f03
more tests
sparkoo Oct 1, 2021
c9dd246
renamed politically incorrect wording
sparkoo Oct 4, 2021
645574e
added more tests to actually test healthz endpoint configuration
sparkoo Oct 4, 2021
27e1ee6
healthz endpoint only on openshift for now
sparkoo Oct 4, 2021
6b747bf
fix when main endpoint is unique
sparkoo Oct 6, 2021
ed44cb8
Merge branch 'main' into gh20481-workspaceHealthzEndpoint
sparkoo Oct 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 76 additions & 115 deletions controllers/devworkspace/solver/che_routing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,8 +297,10 @@ func (c *CheRoutingSolver) getGatewayConfigsAndFillRoutingObjects(cheCluster *v2
configs := make([]corev1.ConfigMap, 0)

// first do routing from main che-gateway into workspace service
if masterConfig := provisionMasterRouting(cheCluster, routing, cmLabels); masterConfig != nil {
configs = append(configs, *masterConfig)
if mainWsRouteConfig, err := provisionMainWorkspaceRoute(cheCluster, routing, cmLabels); err != nil {
return nil, err
} else {
configs = append(configs, *mainWsRouteConfig)
}

// then expose the endpoints
Expand Down Expand Up @@ -346,13 +348,7 @@ func getCommonService(objs *solvers.RoutingObjects, dwId string) *corev1.Service
}

func exposeAllEndpoints(cheCluster *v2alpha1.CheCluster, routing *dwo.DevWorkspaceRouting, objs *solvers.RoutingObjects, ingressExpose func(*EndpointInfo)) *corev1.ConfigMap {
wsRouteConfig := gateway.TraefikConfig{
HTTP: gateway.TraefikConfigHTTP{
Routers: map[string]*gateway.TraefikConfigRouter{},
Services: map[string]*gateway.TraefikConfigService{},
Middlewares: map[string]*gateway.TraefikConfigMiddleware{},
},
}
wsRouteConfig := gateway.CreateEmptyTraefikConfig()

commonService := getCommonService(objs, routing.Spec.DevWorkspaceId)
if commonService == nil {
Expand All @@ -367,7 +363,7 @@ func exposeAllEndpoints(cheCluster *v2alpha1.CheCluster, routing *dwo.DevWorkspa
}

if e.Attributes.GetString(urlRewriteSupportedEndpointAttributeName, nil) == "true" {
addEndpointToTraefikConfig(componentName, e, &wsRouteConfig, cheCluster, routing)
addEndpointToTraefikConfig(componentName, e, wsRouteConfig, cheCluster, routing)
} else {
if !containPort(commonService, int32(e.TargetPort)) {
commonService.Spec.Ports = append(commonService.Spec.Ports, corev1.ServicePort{
Expand Down Expand Up @@ -436,71 +432,33 @@ func containPort(service *corev1.Service, port int32) bool {
return false
}

func provisionMasterRouting(cheCluster *v2alpha1.CheCluster, routing *dwo.DevWorkspaceRouting, cmLabels map[string]string) *corev1.ConfigMap {
cfg := &gateway.TraefikConfig{
HTTP: gateway.TraefikConfigHTTP{
Routers: map[string]*gateway.TraefikConfigRouter{},
Services: map[string]*gateway.TraefikConfigService{},
Middlewares: map[string]*gateway.TraefikConfigMiddleware{},
},
}

rtrs := cfg.HTTP.Routers
srvcs := cfg.HTTP.Services
mdls := cfg.HTTP.Middlewares

func provisionMainWorkspaceRoute(cheCluster *v2alpha1.CheCluster, routing *dwo.DevWorkspaceRouting, cmLabels map[string]string) (*corev1.ConfigMap, error) {
dwId := routing.Spec.DevWorkspaceId
dwNamespace := routing.Namespace

rtrs[dwId] = &gateway.TraefikConfigRouter{
Rule: fmt.Sprintf("PathPrefix(`/%s`)", dwId),
Service: dwId,
Middlewares: calculateMiddlewares(dwId, true),
Priority: 100,
}

srvcs[dwId] = &gateway.TraefikConfigService{
LoadBalancer: gateway.TraefikConfigLoadbalancer{
Servers: []gateway.TraefikConfigLoadbalancerServer{
{
URL: getServiceURL(wsGatewayPort, dwId, dwNamespace),
},
},
},
}

mdls[dwId+"-prefix"] = &gateway.TraefikConfigMiddleware{
StripPrefix: &gateway.TraefikConfigStripPrefix{
Prefixes: []string{"/" + dwId},
},
}
cfg := gateway.CreateCommonTraefikConfig(
dwId,
fmt.Sprintf("PathPrefix(`/%s`)", dwId),
100,
getServiceURL(wsGatewayPort, dwId, dwNamespace),
[]string{"/" + dwId})

if util.IsOpenShift4 {
mdls[dwId+"-auth"] = &gateway.TraefikConfigMiddleware{
ForwardAuth: &gateway.TraefikConfigForwardAuth{
Address: "http://127.0.0.1:8089?namespace=" + dwNamespace,
},
}
// on OpenShift, we need to set authorization header.
// This MUST come before Auth, because Auth needs Authorization header to be properly set.
cfg.AddAuthHeaderRewrite(dwId)

mdls[dwId+"-header"] = &gateway.TraefikConfigMiddleware{
Plugin: &gateway.TraefikPlugin{
HeaderRewrite: &gateway.TraefikPluginHeaderRewrite{
From: "X-Forwarded-Access-Token",
To: "Authorization",
Prefix: "Bearer ",
},
},
}
}
// authorize against kube-rbac-proxy in che-gateway. This will be needed for k8s native auth as well.
cfg.AddAuth(dwId, "http://127.0.0.1:8089?namespace="+dwNamespace)

if len(cfg.HTTP.Routers) > 0 {
contents, err := yaml.Marshal(cfg)
if err != nil {
logger.Error(err, "can't serialize traefik config")
return nil
}
// make '/healthz' path of main endpoints reachable from outside
routeForHealthzEndpoint(cfg, dwId, routing.Spec.Endpoints)
}

configMap := &corev1.ConfigMap{
if contents, err := yaml.Marshal(cfg); err != nil {
return nil, err
} else {
return &corev1.ConfigMap{
ObjectMeta: v1.ObjectMeta{
Name: defaults.GetGatewayWorkspaceConfigMapName(dwId),
Namespace: cheCluster.Namespace,
Expand All @@ -510,24 +468,34 @@ func provisionMasterRouting(cheCluster *v2alpha1.CheCluster, routing *dwo.DevWor
defaults.ConfigAnnotationDevWorkspaceRoutingNamespace: routing.Namespace,
},
},
Data: map[string]string{},
Data: map[string]string{dwId + ".yml": string(contents)},
}, nil
}
}

// makes '/healthz' path of main endpoints reachable from the outside
func routeForHealthzEndpoint(cfg *gateway.TraefikConfig, dwId string, endpoints map[string]dwo.EndpointList) {
for componentName, endpoints := range endpoints {
for _, e := range endpoints {
if e.Attributes.GetString(string(dwo.TypeEndpointAttribute), nil) == string(dwo.MainEndpointType) {
middlewares := []string{dwId + gateway.StripPrefixMiddlewareSuffix}
if util.IsOpenShift4 {
middlewares = append(middlewares, dwId+gateway.HeaderRewriteMiddlewareSuffix)
}
routeName, endpointPath := createEndpointPath(&e, componentName)
cfg.HTTP.Routers[routeName+"-healthz"] = &gateway.TraefikConfigRouter{
Rule: fmt.Sprintf("Path(`/%s%s/healthz`)", dwId, endpointPath),
Service: dwId,
Middlewares: middlewares,
Priority: 101,
}
}
}
configMap.Data[dwId+".yml"] = string(contents)
return configMap
}
return nil
}

func addEndpointToTraefikConfig(componentName string, e dw.Endpoint, cfg *gateway.TraefikConfig, cheCluster *v2alpha1.CheCluster, routing *dwo.DevWorkspaceRouting) {
var routeName string
if e.Attributes.GetString(uniqueEndpointAttributeName, nil) == "true" {
// if endpoint is unique, we're exposing on /componentName/<endpoint-name>
routeName = e.Name
} else {
// if endpoint is NOT unique, we're exposing on /componentName/<port-number>
routeName = strconv.Itoa(e.TargetPort)
}
prefix := fmt.Sprintf("/%s/%s", componentName, routeName)
routeName, prefix := createEndpointPath(&e, componentName)
rulePrefix := fmt.Sprintf("PathPrefix(`%s`)", prefix)

// skip if exact same route is already exposed
Expand All @@ -538,48 +506,41 @@ func addEndpointToTraefikConfig(componentName string, e dw.Endpoint, cfg *gatewa
}

name := fmt.Sprintf("%s-%s-%s", routing.Spec.DevWorkspaceId, componentName, routeName)
cfg.HTTP.Routers[name] = &gateway.TraefikConfigRouter{
Rule: rulePrefix,
Service: name,
Middlewares: calculateMiddlewares(name, false),
Priority: 100,
}

cfg.HTTP.Services[name] = &gateway.TraefikConfigService{
LoadBalancer: gateway.TraefikConfigLoadbalancer{
Servers: []gateway.TraefikConfigLoadbalancerServer{
{
URL: fmt.Sprintf("http://127.0.0.1:%d", e.TargetPort),
},
},
},
}

cfg.HTTP.Middlewares[name+"-prefix"] = &gateway.TraefikConfigMiddleware{
StripPrefix: &gateway.TraefikConfigStripPrefix{
Prefixes: []string{prefix},
},
cfg.AddComponent(
name,
rulePrefix,
100,
fmt.Sprintf("http://127.0.0.1:%d", e.TargetPort),
[]string{prefix})
if util.IsOpenShift4 {
cfg.AddAuth(name, fmt.Sprintf("http://%s.%s:8089?namespace=%s", gateway.GatewayServiceName, cheCluster.Namespace, routing.Namespace))
}

if util.IsOpenShift4 {
cfg.HTTP.Middlewares[name+"-auth"] = &gateway.TraefikConfigMiddleware{
ForwardAuth: &gateway.TraefikConfigForwardAuth{
Address: fmt.Sprintf("http://%s.%s:8089?namespace=%s", gateway.GatewayServiceName, cheCluster.Namespace, routing.Namespace),
},
}
// we need to disable auth for '/healthz' path in main endpoint, for now only on OpenShift
if util.IsOpenShift4 &&
e.Attributes.GetString(string(dwo.TypeEndpointAttribute), nil) == string(dwo.MainEndpointType) {
healthzName := name + "-healthz"
healthzPath := prefix + "/healthz"
cfg.AddComponent(
healthzName,
fmt.Sprintf("Path(`%s`)", healthzPath),
101,
fmt.Sprintf("http://127.0.0.1:%d", e.TargetPort),
[]string{prefix})
}
}

func calculateMiddlewares(name string, header bool) []string {
if util.IsOpenShift4 {
if header {
return []string{name + "-header", name + "-prefix", name + "-auth"}
} else {
return []string{name + "-prefix", name + "-auth"}
}
func createEndpointPath(e *dw.Endpoint, componentName string) (routeName string, path string) {
if e.Attributes.GetString(uniqueEndpointAttributeName, nil) == "true" {
// if endpoint is unique, we're exposing on /componentName/<endpoint-name>
routeName = e.Name
} else {
return []string{name + "-prefix"}
// if endpoint is NOT unique, we're exposing on /componentName/<port-number>
routeName = strconv.Itoa(e.TargetPort)
}
path = fmt.Sprintf("/%s/%s", componentName, routeName)

return routeName, path
}

func findServiceForPort(port int32, objs *solvers.RoutingObjects) *corev1.Service {
Expand Down
Loading