forked from kubeflow/kubeflow
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Prevent deadlocks trying to setup IAP. (kubeflow#924)
* Prevent deadlocks trying to setup IAP.
* We want a single process to update the backend service and IAP.
Right now we do this in the envoy side car which is replicated.
* The script gets a lock to prevent multiple services from trying to update
IAP simultaneously.
* But until the process acquires the lock it can't update the local copy
of the envoy config to do proper JWT validation.
* So we move updating the backend into a separate deployment that has a single
replica 1 (although we still use a lock).
* envoy sidecars now do not modify the backend but just get relevant info
and update the envoy config.
* To support getting the ksonnet app of the bootstrapper add source repository
IAM role to the admin service account and mount the service account
into the bootstrapper.
This will allow use to push the ksonnet app from the bootstrapper to
a cloud source repository.
Related to kubeflow#903
* format jsonnet files.
* Address comments; no need to modify the envoy config.- Loading branch information
1 parent
38a3813
commit 2e384e6
Showing
5 changed files
with
317 additions
and
127 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| #!/bin/bash | ||
| # | ||
| # A script to modify envoy config to perform JWT validation | ||
| # given the information for the service. | ||
| # Script executed by the iap container to configure IAP. When finished, the envoy config is created with the JWT audience. | ||
|
|
||
| [ -z ${CLIENT_ID} ] && echo Error CLIENT_ID must be set && exit 1 | ||
| [ -z ${CLIENT_SECRET} ] && echo Error CLIENT_SECRET must be set && exit 1 | ||
| [ -z ${NAMESPACE} ] && echo Error NAMESPACE must be set && exit 1 | ||
| [ -z ${SERVICE} ] && echo Error SERVICE must be set && exit 1 | ||
|
|
||
| apk add --update jq | ||
| curl https://storage.googleapis.com/kubernetes-release/release/v1.9.4/bin/linux/amd64/kubectl > /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl | ||
|
|
||
|
|
||
| PROJECT=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/project-id) | ||
| if [ -z ${PROJECT} ]; then | ||
| echo Error unable to fetch PROJECT from compute metadata | ||
| exit 1 | ||
| fi | ||
|
|
||
| PROJECT_NUM=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/numeric-project-id) | ||
| if [ -z ${PROJECT_NUM} ]; then | ||
| echo Error unable to fetch PROJECT_NUM from compute metadata | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Activate the service account | ||
| gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} | ||
| # Print out the config for debugging | ||
| gcloud config list | ||
|
|
||
| NODE_PORT=$(kubectl --namespace=${NAMESPACE} get svc ${SERVICE} -o jsonpath='{.spec.ports[0].nodePort}') | ||
| while [[ -z ${BACKEND_ID} ]]; | ||
| do BACKEND_ID=$(gcloud compute --project=${PROJECT} backend-services list --filter=name~k8s-be-${NODE_PORT}- --format='value(id)'); | ||
| echo "Waiting for backend id PROJECT=${PROJECT} NAMESPACE=${NAMESPACE} SERVICE=${SERVICE}..."; | ||
| sleep 2; | ||
| done | ||
| echo BACKEND_ID=${BACKEND_ID} | ||
|
|
||
| NODE_PORT=$(kubectl --namespace=${NAMESPACE} get svc ${SERVICE} -o jsonpath='{.spec.ports[0].nodePort}') | ||
| BACKEND_SERVICE=$(gcloud --project=${PROJECT} compute backend-services list --filter=name~k8s-be-${NODE_PORT}- --uri) | ||
|
|
||
| JWT_AUDIENCE="/projects/${PROJECT_NUM}/global/backendServices/${BACKEND_ID}" | ||
|
|
||
| # For healthcheck compare. | ||
| echo "JWT_AUDIENCE=${JWT_AUDIENCE}" > /var/shared/healthz.env | ||
| echo "NODE_PORT=${NODE_PORT}" >> /var/shared/healthz.env | ||
| echo "BACKEND_ID=${BACKEND_ID}" >> /var/shared/healthz.env | ||
|
|
||
| kubectl get configmap -n ${NAMESPACE} envoy-config -o jsonpath='{.data.envoy-config\.json}' | \ | ||
| sed -e "s|{{JWT_AUDIENCE}}|${JWT_AUDIENCE}|g" > /var/shared/envoy-config.json | ||
|
|
||
| echo "Restarting envoy" | ||
| curl -s ${ENVOY_ADMIN}/quitquitquit | ||
|
|
||
| function checkIAP() { | ||
| # created by init container. | ||
| . /var/shared/healthz.env | ||
|
|
||
| # If node port or backend id change, so does the JWT audience. | ||
| CURR_NODE_PORT=$(kubectl --namespace=${NAMESPACE} get svc ${SERVICE} -o jsonpath='{.spec.ports[0].nodePort}') | ||
| CURR_BACKEND_ID=$(gcloud compute --project=${PROJECT} backend-services list --filter=name~k8s-be-${CURR_NODE_PORT}- --format='value(id)') | ||
| [ "$BACKEND_ID" == "$CURR_BACKEND_ID" ] | ||
| } | ||
|
|
||
| # Verify IAP every 10 seconds. | ||
| while true; do | ||
| if ! checkIAP; then | ||
| echo "$(date) WARN: IAP check failed, restarting container." | ||
| exit 1 | ||
| fi | ||
| sleep 10 | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.