Fix various bugs as part of setting up autodeployments

* Fix some bugs in the blueprints that cropped up while working on
  setting up continuous auto-deployments using the blueprints (GoogleCloudPlatform#5)

Fix some bugs in the documentation.

* Fix bugs in the management config for the per namespace components
  of CNRM. The namespaces of the role bindings wasn't correct so
  the cnrm manager pod ended up not having appropriate permissions.

  * Also the scoped namespace of the cnrm manager statefulset needs
    to be set the managed project not the host project.

* Update Makefile to point at kubeflow/manifests master to pull in cert-manager
changes.

* Add check_domain_length to validate the length of the hostname KF
  deployment name so that we don't end up exceeding the certificate limits.

kubeflow/Makefile

@@ -5,6 +5,8 @@
 MGMTCTXT=$(shell yq r ./instance/settings.yaml mgmt-ctxt)
 # The name of the context for your Kubeflow cluster
 NAME=$(shell yq r ./instance/settings.yaml name)
+PROJECT=$(shell yq r ./instance/settings.yaml project)
+
 KFCTXT=$(NAME)
 
 # Path to kustomize directories
@@ -20,7 +22,7 @@ MANIFESTS_DIR=./upstream/manifests
 BUILD_DIR=.build
 
 # The URL you want to fetch manifests from
-MANIFESTS_URL=https://github.com/jlewi/manifests.git@blueprints
+MANIFESTS_URL=https://github.com/kubeflow/manifests.git@master
 
 # Print out the context
 .PHONY: echo
@@ -29,7 +31,7 @@ echo-ctxt:
 	@echo KFCTXT=$(KFCTXT)
 
 # Get packages
-.PHONY: get-packages
+.PHONY: get-pkg
 get-pkg:
 	# TODO(jlewi): We should switch to using upstream kubeflow/manifests and pin
 	# to a specific version
@@ -80,7 +82,7 @@ apply-kubeflow: hydrate-kubeflow
 # TODO(jlewi): Should we insert appropriate wait statements to wait for various services to
 # be available before continuing?
 .PHONY: apply
-apply: clean-build check-iap apply-gcp wait-gcp create-ctxt apply-asm apply-kubeflow iap-secret
+apply: clean-build check-name check-iap apply-gcp wait-gcp create-ctxt apply-asm apply-kubeflow iap-secret
 
 .PHONY: hydrate-gcp
 hydrate-gcp:
@@ -135,6 +137,11 @@ clean-build:
 hydrate: clean-build hydrate-gcp hydrate-asm hydrate-kubeflow
 
 
+# Make sure the name isn't too long.
+.PHONY: check-name
+check-name:
+	PROJECT=$(PROJECT) NAME=$(NAME) ./hack/check_domain_length.sh
+
 .PHONY: check-iap 
 check-iap:
 	./hack/check_oauth_secret.sh
@@ -158,4 +165,4 @@ create-ctxt:
 
 # Delete gcp resources
 delete-gcp:
-	kubectl	--context=$(MGMTCTXT) delete -f $(BUILD_DIR)/gcp_config
+	kubectl	--context=$(MGMTCTXT) delete -f $(BUILD_DIR)/gcp_config

kubeflow/README.md

@@ -45,7 +45,7 @@ one if you haven't already.
 1. Fetch Kubeflow manifests
 
    ```
-   make pkg-get
+   make get-pkg
    ```
 
   * This generates an error per [GoogleContainerTools/kpt#539](https://github.com/GoogleContainerTools/kpt/issues/539) but it looks like
@@ -65,11 +65,14 @@ one if you haven't already.
    be used to create CNRM resources for your Kubeflow deployment.
 
    ```
-   kpt cfg set mgmt-ctxt
+   kpt cfg set instance mgmt-ctxt ${MANAGEMENT_CONTEXT}
    ```
 
    * Follow the [instructions](../README.md) to create a kubecontext for your managment context
 
+   * **Important** The context must set the namespace to the namespace in your CNRM cluster where you are creating
+     CNRM resources for the managed project. 
+
 1. Pick a name for the Kubeflow deployment
 
    ```

kubeflow/hack/check_domain_length.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# A simple bash script to check that when using CloudEnpoints
+# to create an endpoint we don't endup with a domain
+# that exceeds the maximum allowed length of 62 characters.
+# The domain will be ${NAME}.endpoints.${PROJECT}.cloud.goog\
+#
+# Run this as  PROJECT=${PROJECT} NAME=${NAME} ./check_domain_length
+domain=${NAME}.endpoints.${PROJECT}.cloud.goog
+
+if [ ${#domain} -gt 62 ]; then
+  echo The ${domain} exceeds is ${#domain} characters long which exceeds the maximum length of 62 characters
+  echo choose a shorter name for your deployment
+  exit 1
+fi

kubeflow/instance/gcp_config/kustomization.yaml

@@ -5,12 +5,12 @@ kind: Kustomization
 # namespace should match the project.
 # This assumes we are running CNRM in namespace mode and namespaces match project names.
 namespace: PROJECT # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"gcloud.core.project","value":"PROJECT"}]}}
-commonLabels:
-  kf-name: jlewi-dev
+# TODO(jlewi): do not commit the labels auto-deploy and purpose; they were added
+# as part of autodeployment testing.
+commonLabels:  
+  kf-name: KUBEFLOW-NAME # {"type":"string","x-kustomize":{"setter":{"name":"name","value":"KUBEFLOW-NAME"}}}
 resources:
-- ../../upstream/manifests/gcp/v2/cnrm # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"gcp_manifests_path","value":"../../upstream/manifests"}]}}
+- ../../upstream/manifests/gcp/v2/cnrm
 - iam_policy.yaml
 patchesStrategicMerge:
 - cluster_patch.yaml
-# TODO(jlewi): Base package doesn't currently include the node pool
-# - nodepool_patch.yaml

management/instance/cnrm-install-per-namespace/per-namespace-components.yaml

@@ -24,7 +24,7 @@ metadata:
     cnrm.cloud.google.com/scoped-namespace: MANAGED_PROJECT # {"type":"string","x-kustomize":{"partialSetters":[{"name":"managed_project","value":"MANAGED_PROJECT"}]}}
     cnrm.cloud.google.com/system: "true"
   name: cnrm-admin-binding-MANAGED_PROJECT # {"type":"string","x-kustomize":{"partialSetters":[{"name":"managed_project","value":"MANAGED_PROJECT"}]}}
-  namespace: cnrm-system
+  namespace: MANAGED_PROJECT # {"type":"string","x-kustomize":{"partialSetters":[{"name":"managed_project","value":"MANAGED_PROJECT"}]}}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
@@ -43,7 +43,7 @@ metadata:
     cnrm.cloud.google.com/scoped-namespace: MANAGED_PROJECT # {"type":"string","x-kustomize":{"partialSetters":[{"name":"managed_project","value":"MANAGED_PROJECT"}]}}
     cnrm.cloud.google.com/system: "true"
   name: cnrm-manager-ns-binding-MANAGED_PROJECT # {"type":"string","x-kustomize":{"partialSetters":[{"name":"managed_project","value":"MANAGED_PROJECT"}]}}
-  namespace: cnrm-system
+  namespace: MANAGED_PROJECT # {"type":"string","x-kustomize":{"partialSetters":[{"name":"managed_project","value":"MANAGED_PROJECT"}]}}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
@@ -143,7 +143,7 @@ spec:
     spec:
       containers:
       - args:
-        - --scoped-namespace=HOST_PROJECT # {"type":"string","x-kustomize":{"partialSetters":[{"name":"host_project","value":"HOST_PROJECT"}]}}
+        - --scoped-namespace=MANAGED_PROJECT # {"type":"string","x-kustomize":{"partialSetters":[{"name":"managed_project","value":"MANAGED_PROJECT"}]}}
         - --stderrthreshold=INFO
         - --prometheus-scrape-endpoint=:8888
         command: