diff --git a/CHANGELOG.md b/CHANGELOG.md index da4f4341..0754ad25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,53 @@ The Kubeflow on Google Cloud distribution versioning is following the versioning of [kubeflow/manifests](https://github.com/kubeflow/manifests). -### Unreleased +### v1.6.0-rc.1 + +* Update CHANGELOG (#360) +* Upgrade cert-manger to v1.5.0 (#372) +* Upgrade knative to v1.2 (#373) +* Fix ASM/istio ingress gateway issue (#371) +* Migrate deprecated API calls (#349) +* Remove deprecated KFServing (#375) +* Remove deprecated cloud-endpoints-controller (#377) + +### v1.5.1 + +* Upgrade ASM to v1.13 (#369) +* Fixed KServe issues with dashboard (#362) and directory(#361). +* Increased the maximum length of Kubeflow cluster name (#359). +* Moved RequestAuthentication policy creation to iap-enabler to improve GitOps friendliness (#364). +* Renamed `${name}-kfp-cloudsql` service account into `${name}-sql` to fix the name length restriction (#358) + +### v1.5.0 + +* Upgrade Kubeflow components versions as listed in components versions table +* Integrated with Config Controller, simplified management cluster maintenance cost, there is no need to manually upgrade Config Connector CRD. +* Switch from kfserving to KServe as default serving component, you can switch back to kfserving in config.yaml. +* Fixed cloudsqlproxy issue with livenessProbe configuration. + +### v1.4.1 + +* Upgrade: Integrate with Kubeflow 1.4.1 manifests (kubeflow/manifests#2084) +* Fix: Change cloud endpoint images destination (#343) +* Fix: Use yq4 in iap-ingress Makefile. + +### v1.4.0 + +* Upgrade Kubeflow components versions as listed in components versions table +* Removed GKE 1.18 image version and k8s runtime pin, now GKE version is default to Stable channel. +* Set Emissary Executor as default Argo Workflow executor for Kubeflow Pipelines. +* Upgraded kpt versions from 0.X.X to 1.0.0-beta.6. +* Upgraded yq from v3 to v4. +* Upgraded ASM(Anthos Service Mesh) to 1.10.4-asm.6. +* Unblocked KFSserving usage by removing commonLabels from kustomization patch #298 #324. +* Integrated with KFServing Web App UI. +* Integrated with unified operator: training-operator. +* Simplified deployment: Removed requirement for independent installation of yq, jq, kustomize, kpt. + +### v1.3.1 + +* Change folder name istio-1-9-0 to istio-1-9 ### v1.3.0 diff --git a/kubeflow/apps/.krmignore b/kubeflow/apps/.krmignore index 22f31af0..daee8e6d 100644 --- a/kubeflow/apps/.krmignore +++ b/kubeflow/apps/.krmignore @@ -4,15 +4,12 @@ jupyter/jupyter-web-app/upstream/overlays/istio/params.yaml jupyter/notebook-controller/upstream/crd/kustomizeconfig.yaml katib/upstream/installs/katib-openshift/patches/service-serving-cert.yaml katib/upstream/installs/katib-openshift/patches/webhook-inject-cabundle.yaml -kfserving/upstream/crd/patches/protocol.yaml -kfserving/upstream/overlays/kubeflow/patches/web-app-vsvc.yaml pipelines/upstream/third-party/argo/installs/cluster pipelines/upstream/third-party/argo/installs/namespace pipelines/upstream/third-party/argo/upstream/manifests/namespace-install/overlays admission-webhook/upstream centraldashboard/upstream katib/upstream -kfserving/upstream pipelines/upstream profiles/upstream tensorboard/tensorboard-controller/upstream diff --git a/kubeflow/apps/centraldashboard/kustomization.yaml b/kubeflow/apps/centraldashboard/kustomization.yaml index 76036672..3e56a74b 100644 --- a/kubeflow/apps/centraldashboard/kustomization.yaml +++ b/kubeflow/apps/centraldashboard/kustomization.yaml @@ -2,9 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: kubeflow resources: -# Uncomment this line for kfserving -# - upstream/overlays/istio -# Uncomment this line for kserve - upstream/overlays/kserve patchesStrategicMerge: - deployment-patch.yaml diff --git a/kubeflow/apps/kfserving/Makefile b/kubeflow/apps/kfserving/Makefile deleted file mode 100644 index a62ddbb6..00000000 --- a/kubeflow/apps/kfserving/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -build_dir?=./build -PACKAGE_DIR?=$(shell pwd)/../.. -YQ=docker run --rm -v "$(PACKAGE_DIR)/":/workdir mikefarah/yq:4 -NAME=$(shell $(YQ) e '.data.name' kptconfig/kpt-setter-config.yaml) -PROJECT=$(shell $(YQ) e '.data."gcloud.core.project"' kptconfig/kpt-setter-config.yaml) - -# The kubectl context for your Kubeflow cluster -KFCTXT=$(NAME) - -.PHONY: apply -apply: hydrate -# Apply App kfserving - kubectl --context=$(KFCTXT) apply -f $(build_dir) - kubectl --context=$(KFCTXT) patch cm config-domain --namespace knative-serving --type merge -p '{"data":{"$(NAME).endpoints.$(PROJECT).cloud.goog": ""}}' - -.PHONY: hydrate -hydrate: -# Hydrate App kfserving - rm -rf $(build_dir) && mkdir -p $(build_dir) - kustomize build -o $(build_dir) ./ diff --git a/kubeflow/apps/kfserving/README.md b/kubeflow/apps/kfserving/README.md deleted file mode 100644 index 5626e17e..00000000 --- a/kubeflow/apps/kfserving/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# KFServing usage guide - -KFServing has been renamed to kserve. Refer to `GCP-IAP` example in https://github.com/kserve/kserve/tree/master/docs/samples/gcp-iap for creating inference service and accessing such endpoint. diff --git a/kubeflow/apps/kfserving/deployment-patch.yaml b/kubeflow/apps/kfserving/deployment-patch.yaml deleted file mode 100644 index f957436a..00000000 --- a/kubeflow/apps/kfserving/deployment-patch.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kfserving-models-web-app - namespace: kfserving-system -spec: - template: - spec: - containers: - - name: kfserving-models-web-app - env: - - name: USERID_HEADER - value: null - valueFrom: - configMapKeyRef: - name: kubeflow-config - key: userid-header - - name: USERID_PREFIX - value: null - valueFrom: - configMapKeyRef: - name: kubeflow-config - key: userid-prefix diff --git a/kubeflow/apps/kfserving/kustomization.yaml b/kubeflow/apps/kfserving/kustomization.yaml deleted file mode 100644 index e35b3466..00000000 --- a/kubeflow/apps/kfserving/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: kubeflow -resources: -- upstream/overlays/kubeflow -patchesStrategicMerge: -- deployment-patch.yaml diff --git a/kubeflow/apps/pipelines/pull-upstream.sh b/kubeflow/apps/pipelines/pull-upstream.sh index 756c1b26..fb5c3f9f 100755 --- a/kubeflow/apps/pipelines/pull-upstream.sh +++ b/kubeflow/apps/pipelines/pull-upstream.sh @@ -17,7 +17,7 @@ set -ex # TODO: Use kubeflow/pipelines once https://github.com/kubeflow/pipelines/pull/6595 is resolved. -export KUBEFLOW_PIPELINES_VERSION=1.8.2 +export KUBEFLOW_PIPELINES_VERSION=2.0.0-alpha.3 export KUBEFLOW_PIPELINES_REPO=https://github.com/kubeflow/pipelines.git # export KUBEFLOW_PIPELINES_VERSION=upgradekpt # Other attempted branches: krmignore, kubeflow14 # export KUBEFLOW_PIPELINES_REPO=https://github.com/zijianjoy/pipelines.git diff --git a/kubeflow/common/.krmignore b/kubeflow/common/.krmignore index 91777c07..36b2cf75 100644 --- a/kubeflow/common/.krmignore +++ b/kubeflow/common/.krmignore @@ -7,3 +7,4 @@ istio/upstream/profile-overlay.yaml istio/upstream/profile.yaml user-namespace/upstream/base/params.yaml knative/upstream/knative-eventing-post-install-jobs/base/eventing-post-install-jobs.yaml +iap-ingress/base/swagger_template.yaml diff --git a/kubeflow/common/cloud-endpoints/.krmignore b/kubeflow/common/cloud-endpoints/.krmignore deleted file mode 100644 index ab0fcbca..00000000 --- a/kubeflow/common/cloud-endpoints/.krmignore +++ /dev/null @@ -1 +0,0 @@ -base/params.yaml diff --git a/kubeflow/common/cloud-endpoints/Kptfile b/kubeflow/common/cloud-endpoints/Kptfile deleted file mode 100755 index b2249d71..00000000 --- a/kubeflow/common/cloud-endpoints/Kptfile +++ /dev/null @@ -1,249 +0,0 @@ -apiVersion: kpt.dev/v1 -kind: Kptfile -metadata: - name: instance -info: - description: sample description -# openAPI: -# definitions: -# io.k8s.cli.substitutions.project: -# x-k8s-cli: -# substitution: -# name: project -# pattern: ${gcloud.core.project} -# values: -# - marker: ${gcloud.core.project} -# ref: '#/definitions/io.k8s.cli.setters.gcloud.core.project' -# io.k8s.cli.setters.gcloud.core.project: -# x-k8s-cli: -# setter: -# name: gcloud.core.project -# value: jamxl-kfp-dev -# isSet: true -# io.k8s.cli.substitutions.name-label: -# x-k8s-cli: -# substitution: -# name: name-label -# pattern: ${name} -# values: -# - marker: ${name} -# ref: '#/definitions/io.k8s.cli.setters.name' -# io.k8s.cli.setters.name: -# x-k8s-cli: -# setter: -# name: name -# value: kf-ref19 -# isSet: true -# io.k8s.cli.setters.location: -# x-k8s-cli: -# setter: -# name: location -# value: "us-west1-b" -# isSet: true -# io.k8s.cli.setters.mgmt-ctxt: -# x-k8s-cli: -# setter: -# name: mgmt-ctxt -# value: kf-man2 -# isSet: true -# io.k8s.cli.setters.kustomize_manifests_path: -# x-k8s-cli: -# setter: -# name: kustomize_manifests_path -# value: ../../../upstream/manifests -# io.k8s.cli.substitutions.knative-installs-generic: -# x-k8s-cli: -# substitution: -# name: knative-installs-generic -# pattern: ${kustomize_manifests_path}/knative/installs/generic -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.gcp-iap-ingress-v3: -# x-k8s-cli: -# substitution: -# name: gcp-iap-ingress-v3 -# pattern: ${kustomize_manifests_path}/gcp/iap-ingress/v3 -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.namespaces-base: -# x-k8s-cli: -# substitution: -# name: namespaces-base -# pattern: ${kustomize_manifests_path}/namespaces/base -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.istio-iap-gateway-base: -# x-k8s-cli: -# substitution: -# name: istio-iap-gateway-base -# pattern: ${kustomize_manifests_path}/istio/iap-gateway/base -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.metacontroller-base: -# x-k8s-cli: -# substitution: -# name: metacontroller-base -# pattern: ${kustomize_manifests_path}/metacontroller/base -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.cert-manager-cert-manager-kube-system-resources-base: -# x-k8s-cli: -# substitution: -# name: cert-manager-cert-manager-kube-system-resources-base -# pattern: ${kustomize_manifests_path}/cert-manager/cert-manager-kube-system-resources/base -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.cert-manager-cert-manager-v3: -# x-k8s-cli: -# substitution: -# name: cert-manager-cert-manager-v3 -# pattern: ${kustomize_manifests_path}/cert-manager/cert-manager/v3 -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.istio-istio-base: -# x-k8s-cli: -# substitution: -# name: istio-istio-base -# pattern: ${kustomize_manifests_path}/istio/istio/base -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.application-v3: -# x-k8s-cli: -# substitution: -# name: application-v3 -# pattern: ${kustomize_manifests_path}/application/v3 -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.stackds-gcp: -# x-k8s-cli: -# substitution: -# name: stackds-gcp -# pattern: ${kustomize_manifests_path}/stackds/gcp -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.gcp-cloud-endpoints-overlays-application: -# x-k8s-cli: -# substitution: -# name: gcp-cloud-endpoints-overlays-application -# pattern: ${kustomize_manifests_path}/gcp/cloud-endpoints/overlays/application -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.cert-manager-cert-manager-cdrds-base: -# x-k8s-cli: -# substitution: -# name: cert-manager-cert-manager-cdrds-base -# pattern: ${kustomize_manifests_path}/cert-manager/cert-manager-cdrds/base -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.cert-manager-cert-manager-kubeflow-issuer: -# x-k8s-cli: -# substitution: -# name: cert-manager-cert-manager-kubeflow-issuer -# pattern: ${kustomize_manifests_path}/cert-manager/cert-manager/kubeflow-issuer -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.substitutions.stacks-gcp: -# x-k8s-cli: -# substitution: -# name: stacks-gcp -# pattern: ${kustomize_manifests_path}/stacks/gcp -# values: -# - marker: ${kustomize_manifests_path} -# ref: '#/definitions/io.k8s.cli.setters.kustomize_manifests_path' -# io.k8s.cli.setters.gke.private: -# x-k8s-cli: -# setter: -# name: gke.private -# value: "false" -# isSet: true -# io.k8s.cli.setters.email: -# x-k8s-cli: -# setter: -# name: email -# value: jamxl@google.com -# isSet: true -# io.k8s.cli.substitutions.gcp-sa: -# x-k8s-cli: -# substitution: -# name: gcp-sa -# pattern: ${name}-user@${gcloud.core.project}.iam.gserviceaccount.com -# values: -# - marker: ${name} -# ref: '#/definitions/io.k8s.cli.setters.name' -# - marker: ${gcloud.core.project} -# ref: '#/definitions/io.k8s.cli.setters.gcloud.core.project' -# io.k8s.cli.substitutions.ip-name: -# x-k8s-cli: -# substitution: -# name: ip-name -# pattern: ${name}-ip -# values: -# - marker: ${name} -# ref: '#/definitions/io.k8s.cli.setters.name' -# io.k8s.cli.substitutions.hostname: -# x-k8s-cli: -# substitution: -# name: hostname -# pattern: ${name}.endpoints.${gcloud.core.project}.cloud.goog -# values: -# - marker: ${name} -# ref: '#/definitions/io.k8s.cli.setters.name' -# - marker: ${gcloud.core.project} -# ref: '#/definitions/io.k8s.cli.setters.gcloud.core.project' -# io.k8s.cli.substitutions.storage-artifact-store-name: -# x-k8s-cli: -# substitution: -# name: storage-artifact-store-name -# pattern: ${name}-storage-artifact-store -# values: -# - marker: ${name} -# ref: '#/definitions/io.k8s.cli.setters.name' -# io.k8s.cli.substitutions.metadata-artifact-store-name: -# x-k8s-cli: -# substitution: -# name: metadata-artifact-store-name -# pattern: ${name}-storage-metadata-store -# values: -# - marker: ${name} -# ref: '#/definitions/io.k8s.cli.setters.name' -# io.k8s.cli.substitutions.gcp-sa-admin: -# x-k8s-cli: -# substitution: -# name: gcp-sa-admin -# pattern: ${name}-admin@${gcloud.core.project}.iam.gserviceaccount.com -# values: -# - marker: ${name} -# ref: '#/definitions/io.k8s.cli.setters.name' -# - marker: ${gcloud.core.project} -# ref: '#/definitions/io.k8s.cli.setters.gcloud.core.project' -# io.k8s.cli.substitutions.vm-sa-ref: -# x-k8s-cli: -# substitution: -# name: vm-sa-ref -# pattern: ${name}-vm@${gcloud.core.project}.iam.gserviceaccount.com -# values: -# - marker: ${name} -# ref: '#/definitions/io.k8s.cli.setters.name' -# - marker: ${gcloud.core.project} -# ref: '#/definitions/io.k8s.cli.setters.gcloud.core.project' -# io.k8s.cli.substitutions.node-pool-example: -# x-k8s-cli: -# substitution: -# name: node-pool-example -# pattern: ${name}-example -# values: -# - marker: ${name} -# ref: '#/definitions/io.k8s.cli.setters.name' diff --git a/kubeflow/common/cloud-endpoints/base/cluster-role-binding.yaml b/kubeflow/common/cloud-endpoints/base/cluster-role-binding.yaml deleted file mode 100755 index 79ff5afd..00000000 --- a/kubeflow/common/cloud-endpoints/base/cluster-role-binding.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: cloud-endpoints-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cloud-endpoints-controller -subjects: -- kind: ServiceAccount - name: kf-admin diff --git a/kubeflow/common/cloud-endpoints/base/cluster-role.yaml b/kubeflow/common/cloud-endpoints/base/cluster-role.yaml deleted file mode 100755 index fac2877b..00000000 --- a/kubeflow/common/cloud-endpoints/base/cluster-role.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: cloud-endpoints-controller -rules: -- apiGroups: - - "" - resources: - - services - - configmaps - verbs: - - get - - list -- apiGroups: - - extensions - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list diff --git a/kubeflow/common/cloud-endpoints/base/composite-controller.yaml b/kubeflow/common/cloud-endpoints/base/composite-controller.yaml deleted file mode 100755 index 43146ad5..00000000 --- a/kubeflow/common/cloud-endpoints/base/composite-controller.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: metacontroller.k8s.io/v1alpha1 -kind: CompositeController -metadata: - name: cloud-endpoints-controller -spec: - childResources: [] - clientConfig: - service: - caBundle: '...' - name: cloud-endpoints-controller - namespace: $(namespace) - generateSelector: true - hooks: - sync: - webhook: - url: http://cloud-endpoints-controller.$(namespace)/sync - parentResource: - apiVersion: ctl.isla.solutions/v1 - resource: cloudendpoints - resyncPeriodSeconds: 2 diff --git a/kubeflow/common/cloud-endpoints/base/crd.yaml b/kubeflow/common/cloud-endpoints/base/crd.yaml deleted file mode 100755 index 4d09e9fb..00000000 --- a/kubeflow/common/cloud-endpoints/base/crd.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cloudendpoints.ctl.isla.solutions -spec: - group: ctl.isla.solutions - names: - kind: CloudEndpoint - plural: cloudendpoints - shortNames: - - cloudep - - ce - singular: cloudendpoint - scope: Namespaced - version: v1 diff --git a/kubeflow/common/cloud-endpoints/base/deployment.yaml b/kubeflow/common/cloud-endpoints/base/deployment.yaml deleted file mode 100755 index e3c44dd9..00000000 --- a/kubeflow/common/cloud-endpoints/base/deployment.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cloud-endpoints-controller -spec: - replicas: 1 - template: - metadata: - labels: - app: cloud-endpoints-controller - annotations: - sidecar.istio.io/inject: "false" - spec: - containers: - - image: gcr.io/ml-pipeline/cloud-solutions-group/cloud-endpoints-controller:0.2.1 - imagePullPolicy: Always - name: cloud-endpoints-controller - readinessProbe: - failureThreshold: 2 - httpGet: - path: /healthz - port: 80 - scheme: HTTP - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 5 - serviceAccountName: kf-admin - terminationGracePeriodSeconds: 5 diff --git a/kubeflow/common/cloud-endpoints/base/gcp-credentials-patch.yaml b/kubeflow/common/cloud-endpoints/base/gcp-credentials-patch.yaml deleted file mode 100755 index 02daa7f7..00000000 --- a/kubeflow/common/cloud-endpoints/base/gcp-credentials-patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Patch the env/volumes/volumeMounts for GCP credentials -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cloud-endpoints-controller -spec: - template: - spec: - containers: - - name: cloud-endpoints-controller - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /var/run/secrets/sa/admin-gcp-sa.json - volumeMounts: - - mountPath: /var/run/secrets/sa - name: sa-key - readOnly: true - volumes: - - name: sa-key - secret: - secretName: admin-gcp-sa diff --git a/kubeflow/common/cloud-endpoints/base/kustomization.yaml b/kubeflow/common/cloud-endpoints/base/kustomization.yaml deleted file mode 100755 index bc4370a0..00000000 --- a/kubeflow/common/cloud-endpoints/base/kustomization.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: kubeflow -resources: -- cluster-role-binding.yaml -- cluster-role.yaml -- composite-controller.yaml -- crd.yaml -- deployment.yaml -- service-account.yaml -- service.yaml -commonLabels: - app: cloud-endpoints-controller - kustomize.component: cloud-endpoints -images: -- name: gcr.io/ml-pipeline/cloud-solutions-group/cloud-endpoints-controller - newName: gcr.io/ml-pipeline/cloud-solutions-group/cloud-endpoints-controller - newTag: 0.2.1 -configMapGenerator: -- name: cloud-endpoints-parameters - envs: - - params.env -generatorOptions: - disableNameSuffixHash: true -vars: -- name: secretName - objref: - kind: ConfigMap - name: cloud-endpoints-parameters - apiVersion: v1 - fieldref: - fieldpath: data.secretName -- name: namespace - objref: - kind: ConfigMap - name: cloud-endpoints-parameters - apiVersion: v1 - fieldref: - fieldpath: data.namespace -configurations: -- params.yaml diff --git a/kubeflow/common/cloud-endpoints/base/params.env b/kubeflow/common/cloud-endpoints/base/params.env deleted file mode 100755 index 53ba6bd1..00000000 --- a/kubeflow/common/cloud-endpoints/base/params.env +++ /dev/null @@ -1,2 +0,0 @@ -namespace=kubeflow -secretName=admin-gcp-sa diff --git a/kubeflow/common/cloud-endpoints/base/params.yaml b/kubeflow/common/cloud-endpoints/base/params.yaml deleted file mode 100755 index 61954abf..00000000 --- a/kubeflow/common/cloud-endpoints/base/params.yaml +++ /dev/null @@ -1,7 +0,0 @@ -varReference: -- path: spec/template/spec/volumes/secret/secretName - kind: Deployment -- path: spec/clientConfig/service/namespace - kind: CompositeController -- path: spec/hooks/sync/webhook/url - kind: CompositeController diff --git a/kubeflow/common/cloud-endpoints/base/service-account.yaml b/kubeflow/common/cloud-endpoints/base/service-account.yaml deleted file mode 100755 index ce1417d6..00000000 --- a/kubeflow/common/cloud-endpoints/base/service-account.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kf-admin diff --git a/kubeflow/common/cloud-endpoints/base/service.yaml b/kubeflow/common/cloud-endpoints/base/service.yaml deleted file mode 100755 index 5d0b3de7..00000000 --- a/kubeflow/common/cloud-endpoints/base/service.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: cloud-endpoints-controller -spec: - ports: - - name: http - port: 80 - selector: - app: cloud-endpoints-controller - type: ClusterIP diff --git a/kubeflow/common/cloud-endpoints/kustomization.yaml b/kubeflow/common/cloud-endpoints/kustomization.yaml deleted file mode 100755 index f49f037e..00000000 --- a/kubeflow/common/cloud-endpoints/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -bases: -- ./base -patchesStrategicMerge: -- service-accounts.yaml diff --git a/kubeflow/common/cloud-endpoints/overlays/application/application.yaml b/kubeflow/common/cloud-endpoints/overlays/application/application.yaml deleted file mode 100755 index f7017051..00000000 --- a/kubeflow/common/cloud-endpoints/overlays/application/application.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: app.k8s.io/v1beta1 -kind: Application -metadata: - name: cloud-endpoints -spec: - selector: - matchLabels: - app.kubernetes.io/name: cloud-endpoints - app.kubernetes.io/instance: cloud-endpoints-v0.7.0 - app.kubernetes.io/managed-by: kfctl - app.kubernetes.io/component: cloud-endpoints - app.kubernetes.io/part-of: kubeflow - app.kubernetes.io/version: v0.7.0 - componentKinds: - - group: core - kind: ConfigMap - - group: apps - kind: Deployment - descriptor: - type: cloud-endpoints - version: v1beta1 - description: "" - maintainers: [] - owners: [] - keywords: - - cloud-endpoints - - kubeflow - links: - - description: About - url: "" - addOwnerRef: true diff --git a/kubeflow/common/cloud-endpoints/overlays/application/kustomization.yaml b/kubeflow/common/cloud-endpoints/overlays/application/kustomization.yaml deleted file mode 100755 index 844385ce..00000000 --- a/kubeflow/common/cloud-endpoints/overlays/application/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -bases: -- ../../base -commonLabels: - app.kubernetes.io/component: cloud-endpoints - app.kubernetes.io/name: cloud-endpoints -kind: Kustomization -namespace: kubeflow -resources: -- application.yaml diff --git a/kubeflow/common/cloud-endpoints/overlays/gcp-credentials/gcp-credentials-patch.yaml b/kubeflow/common/cloud-endpoints/overlays/gcp-credentials/gcp-credentials-patch.yaml deleted file mode 100755 index 02daa7f7..00000000 --- a/kubeflow/common/cloud-endpoints/overlays/gcp-credentials/gcp-credentials-patch.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Patch the env/volumes/volumeMounts for GCP credentials -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cloud-endpoints-controller -spec: - template: - spec: - containers: - - name: cloud-endpoints-controller - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /var/run/secrets/sa/admin-gcp-sa.json - volumeMounts: - - mountPath: /var/run/secrets/sa - name: sa-key - readOnly: true - volumes: - - name: sa-key - secret: - secretName: admin-gcp-sa diff --git a/kubeflow/common/cloud-endpoints/overlays/gcp-credentials/kustomization.yaml b/kubeflow/common/cloud-endpoints/overlays/gcp-credentials/kustomization.yaml deleted file mode 100755 index 820285a9..00000000 --- a/kubeflow/common/cloud-endpoints/overlays/gcp-credentials/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -bases: -- ../../base -patchesStrategicMerge: -- gcp-credentials-patch.yaml diff --git a/kubeflow/common/cloud-endpoints/service-accounts.yaml b/kubeflow/common/cloud-endpoints/service-accounts.yaml deleted file mode 100755 index ebcc345a..00000000 --- a/kubeflow/common/cloud-endpoints/service-accounts.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kf-admin - annotations: - iam.gke.io/gcp-service-account: ${name}-admin@${gcloud.core.project}.iam.gserviceaccount.com # kpt-set: ${name}-admin@${gcloud.core.project}.iam.gserviceaccount.com diff --git a/kubeflow/common/iap-ingress/Makefile b/kubeflow/common/iap-ingress/Makefile index 2d0d7507..de68f398 100644 --- a/kubeflow/common/iap-ingress/Makefile +++ b/kubeflow/common/iap-ingress/Makefile @@ -30,3 +30,5 @@ pod-reset: # Kick the backend updater pod, because information might be outdated after the apply. # https://github.com/kubeflow/gcp-blueprints/issues/160 kubectl --context=$(KFCTXT) -n istio-system delete statefulset backend-updater + # Kick the cloud-endpoints-enabler deployment + kubectl --context=$(KFCTXT) -n istio-system delete deployment cloud-endpoints-enabler diff --git a/kubeflow/common/iap-ingress/README.md b/kubeflow/common/iap-ingress/README.md new file mode 100644 index 00000000..a5bf0438 --- /dev/null +++ b/kubeflow/common/iap-ingress/README.md @@ -0,0 +1,19 @@ +# Identity-Aware Proxy (IAP) Ingress + +[IAP](https://cloud.google.com/iap) establishes a central authorization layer via HTTPS and enables application-level access control. Your kubeflow cluster can only be accessed through the proxy by users, who have the correct Identity and Access Management (IAM) role. When you grant a user access by IAP, they're subject to the fine-grained access controls without requiring a VPN. When a user tries to access the kubeflow cluster, IAP performs authentication and authorization checks. + +IAP is [integrated through Ingress](https://cloud.google.com/iap/docs/enabling-kubernetes-howto). The incoming traffic is handled by [HTTPS Load Balancing](https://cloud.google.com/load-balancing/docs/https), a component of Cloud Load Balancing, configured by the Ingress controller. The Ingress controller gets configuration information from an [Ingress](https://cloud.google.com/kubernetes-engine/docs/concepts/ingress) object (**envoy-ingress**) that is associated with one or more Service objects. Each Service object holds routing information that is used to direct an incoming request to a particular Pod and port. The Ingress controller reads configuration information from the BackendConfig (**iap-backendconfig**) and sets up the load balancer accordingly. **iap-backendconfig** holds configuration information that is specific to Cloud Load Balancing. + +To create a fully qualified domain name (FQDN) for the kubeflow cluster and expose it through HTTPS, we employ [Cloud Endpoints](https://cloud.google.com/endpoints). Cloud Endpoints is an API management system that helps you secure, monitor, analyze, and set quotas on your APIs using the same infrastructure Google uses for its own APIs. Endpoints works with the Extensible Service Proxy (ESP) and the Extensible Service Proxy V2 (ESPv2) to provide API management. Endpoints supports version 2 of the OpenAPI Specification (formerly known as the [Swagger spec](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/2.0.md)) — the industry standard for defining REST APIs. If you are unfamiliar with the OpenAPI Specification, see [OpenAPI Overview](https://cloud.google.com/endpoints/docs/openapi/openapi-overview). + +## iap-enabler + +[IAP uses](https://cloud.google.com/iap/docs/signed-headers-howto) JSON Web Tokens ([JWT](https://jwt.io/introduction)) to make sure that a request to kubeflow is authorized. This protects kubeflow from IAP being accidentally disabled, misconfigured firewalls, and access from within the project. This *Deployment* applies a RequestAuthentication (**ingress-jwt**) to the kubeflow cluster based on the [policy.yaml template](./base/policy.yaml). + +## backend-updater + +HTTPS Load Balancing requires a [health check](https://cloud.google.com/load-balancing/docs/health-check-concepts) to determine if backend (**istio-ingressgateway**) responds to traffic. This *StatefulSet* updates the **iap-backendconfig** with the appropriate backend port and backend path for the corresponding health check. + +## cloud-endpoints-enabler + +This *Deployment* is introduced to replace cloud-endpoints-controller. It [establishes a cloud endpoint](https://cloud.google.com/endpoints/docs/openapi/get-started-kubernetes-engine-espv2) using OpenAPI specification. It uses [swagger_template.yaml](./base/swagger_template.yaml) to build an appropriate OpenAPI spec. This template was used in the original [cloud-endpoint-controller](https://github.com/danisla/cloud-endpoints-controller) (deprecated) in Kubeflow 1.5.1 and earlier. diff --git a/kubeflow/common/iap-ingress/base/cloud-endpoint.yaml b/kubeflow/common/iap-ingress/base/cloud-endpoint.yaml deleted file mode 100755 index 139273b2..00000000 --- a/kubeflow/common/iap-ingress/base/cloud-endpoint.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: ctl.isla.solutions/v1 -kind: CloudEndpoint -metadata: - name: $(appName) -spec: - project: $(project) - targetIngress: - name: $(ingressName) - namespace: $(istioNamespace) diff --git a/kubeflow/common/iap-ingress/base/cluster-role-binding.yaml b/kubeflow/common/iap-ingress/base/cluster-role-binding.yaml index b1905032..9d0390d6 100755 --- a/kubeflow/common/iap-ingress/base/cluster-role-binding.yaml +++ b/kubeflow/common/iap-ingress/base/cluster-role-binding.yaml @@ -1,4 +1,4 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: kf-admin-iap diff --git a/kubeflow/common/iap-ingress/base/cluster-role.yaml b/kubeflow/common/iap-ingress/base/cluster-role.yaml index 67d36eba..690f0ec7 100755 --- a/kubeflow/common/iap-ingress/base/cluster-role.yaml +++ b/kubeflow/common/iap-ingress/base/cluster-role.yaml @@ -1,4 +1,4 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: kf-admin-iap diff --git a/kubeflow/common/iap-ingress/base/config-map.yaml b/kubeflow/common/iap-ingress/base/config-map.yaml index 653ca341..9c782239 100644 --- a/kubeflow/common/iap-ingress/base/config-map.yaml +++ b/kubeflow/common/iap-ingress/base/config-map.yaml @@ -1,5 +1,3 @@ -# TODO(jlewi): We should switch to using a configmap generator to load from external files. ---- apiVersion: v1 data: healthcheck_route.yaml: | @@ -181,6 +179,92 @@ data: echo "Backend updated successfully. Waiting 1 hour before updating again." sleep 3600 done + setup_cloudendpoints.sh: | + #!/bin/bash + # + # A simple shell script to configure a cloud endpoint + set -x + [ -z ${NAMESPACE} ] && echo Error NAMESPACE must be set && exit 1 + [ -z ${SERVICE} ] && echo Error SERVICE must be set && exit 1 + [ -z ${INGRESS_NAME} ] && echo Error INGRESS_NAME must be set && exit 1 + [ -z ${ENDPOINT_NAME} ] && echo Error ENDPOINT_NAME must be set && exit 1 + + __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + + PROJECT=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/project-id) + if [ -z ${PROJECT} ]; then + echo Error unable to fetch PROJECT from compute metadata + exit 1 + fi + + PROJECT_NUM=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/numeric-project-id) + if [ -z ${PROJECT_NUM} ]; then + echo Error unable to fetch PROJECT_NUM from compute metadata + exit 1 + fi + + # Activate the service account + if [ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + # As of 0.7.0 we should be using workload identity and never setting GOOGLE_APPLICATION_CREDENTIALS. + # But we kept this for backwards compatibility but can remove later. + gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS} + fi + + # Print out the config for debugging + gcloud config list + gcloud auth list + + set_endpoint () { + NODE_PORT=$(kubectl --namespace=${NAMESPACE} get svc ${SERVICE} -o jsonpath='{.spec.ports[?(@.name=="http2")].nodePort}') + echo "[DEBUG] node port is ${NODE_PORT}" + + BACKEND_NAME="" + while [[ -z ${BACKEND_NAME} ]]; do + BACKENDS=$(kubectl --namespace=${NAMESPACE} get ingress ${INGRESS_NAME} -o jsonpath='{.metadata.annotations.ingress\.kubernetes\.io/backends}') + echo "[DEBUG] fetching backends info with ${INGRESS_NAME}: ${BACKENDS}" + BACKEND_NAME=$(echo $BACKENDS | grep -o "k8s-be-${NODE_PORT}--[0-9a-z]\+") + echo "[DEBUG] backend name is ${BACKEND_NAME}" + sleep 2 + done + + BACKEND_ID="" + while [[ -z ${BACKEND_ID} ]]; do + BACKEND_ID=$(gcloud compute --project=${PROJECT} backend-services list --filter=name~${BACKEND_NAME} --format='value(id)') + echo "[DEBUG] Waiting for backend id PROJECT=${PROJECT} NAMESPACE=${NAMESPACE} SERVICE=${SERVICE} filter=name~${BACKEND_NAME}" + sleep 2 + done + echo BACKEND_ID=${BACKEND_ID} + + JWT_AUDIENCE="/projects/${PROJECT_NUM}/global/backendServices/${BACKEND_ID}" + + # We use a regular expression to obtain the IP address of the target Ingress, assuming IPv4 standard. + INGRESS_TARGET_IP=$(kubectl get ingress --all-namespaces | grep -E -o "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)") + + echo "[DEBUG] ENDPOINT_NAME = ${ENDPOINT_NAME}" + echo "[DEBUG] INGRESS_TARGET_IP = ${INGRESS_TARGET_IP}" + echo "[DEBUG] JWT_AUDIENCE = ${JWT_AUDIENCE}" + + # Create OpenAPI specification for the RESTful Cloud Endpoint + sed "s|JWT_AUDIENCE|${JWT_AUDIENCE}|;s|ENDPOINT_NAME|${ENDPOINT_NAME}|;s|INGRESS_TARGET_IP|${INGRESS_TARGET_IP}|" /var/envoy-config/swagger_template.yaml > openapi.yaml + + # Deploy and enable the endpoint + gcloud endpoints services deploy openapi.yaml + gcloud services enable ${ENDPOINT_NAME} + + # Create IAM resources used by the endpoint + gcloud endpoints services add-iam-policy-binding ${ENDPOINT_NAME} \ + --member serviceAccount:${SERVICE_ACCOUNTNAME} \ + --role roles/servicemanagement.serviceController + gcloud projects add-iam-policy-binding ${PROJECT} \ + --member serviceAccount:${SERVICE_ACCOUNTNAME} \ + --role roles/cloudtrace.agent + } + + while true; do + set_endpoint + echo "Sleeping 30 seconds..." + sleep 30 + done kind: ConfigMap metadata: name: envoy-config @@ -212,4 +296,3 @@ data: kind: ConfigMap metadata: name: ingress-bootstrap-config ---- diff --git a/kubeflow/common/iap-ingress/base/deployment.yaml b/kubeflow/common/iap-ingress/base/deployment.yaml index 6ccb8658..e3131206 100755 --- a/kubeflow/common/iap-ingress/base/deployment.yaml +++ b/kubeflow/common/iap-ingress/base/deployment.yaml @@ -64,3 +64,43 @@ spec: - configMap: name: envoy-config name: config-volume +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cloud-endpoints-enabler +spec: + replicas: 1 + template: + metadata: + labels: + service: cloud-endpoints-enabler + spec: + containers: + - command: + - bash + - /var/envoy-config/setup_cloudendpoints.sh + env: + - name: NAMESPACE + value: $(istioNamespace) + - name: SERVICE + value: istio-ingressgateway + - name: INGRESS_NAME + value: $(ingressName) + - name: USE_ISTIO + value: "true" + - name: ENDPOINT_NAME + value: $(hostname) + - name: SERVICE_ACCOUNTNAME + value: $(appName)-admin@$(project).iam.gserviceaccount.com + image: gcr.io/kubeflow-images-public/ingress-setup:latest + name: cloud-endpoints-enabler + volumeMounts: + - mountPath: /var/envoy-config/ + name: config-volume + restartPolicy: Always + serviceAccountName: kf-admin + volumes: + - configMap: + name: envoy-config + name: config-volume diff --git a/kubeflow/common/iap-ingress/base/ingress.yaml b/kubeflow/common/iap-ingress/base/ingress.yaml index 6afcfc6d..65f6f7b5 100755 --- a/kubeflow/common/iap-ingress/base/ingress.yaml +++ b/kubeflow/common/iap-ingress/base/ingress.yaml @@ -1,20 +1,25 @@ -apiVersion: networking.k8s.io/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: - kubernetes.io/ingress.allow-http: "false" + kubernetes.io/ingress.allow-http: "false" kubernetes.io/ingress.global-static-ip-name: $(ipName) networking.gke.io/managed-certificates: gke-certificate name: envoy-ingress spec: - backend: - serviceName: istio-ingressgateway - servicePort: 80 + defaultBackend: + service: + name: istio-ingressgateway + port: + number: 80 rules: - host: $(hostname) http: paths: - backend: - serviceName: istio-ingressgateway - servicePort: 80 + service: + name: istio-ingressgateway + port: + number: 80 path: /* + pathType: ImplementationSpecific diff --git a/kubeflow/common/iap-ingress/base/kustomization.yaml b/kubeflow/common/iap-ingress/base/kustomization.yaml index ea298313..4a54b7f0 100755 --- a/kubeflow/common/iap-ingress/base/kustomization.yaml +++ b/kubeflow/common/iap-ingress/base/kustomization.yaml @@ -3,7 +3,6 @@ kind: Kustomization resources: - backend-config.yaml - cert.yaml -- cloud-endpoint.yaml - cluster-role-binding.yaml - cluster-role.yaml - config-map.yaml @@ -27,10 +26,11 @@ images: newName: gcr.io/ml-pipeline/cloud-solutions-group/esp-sample-app newTag: 1.0.0 configMapGenerator: - - name: envoy-config - behavior: merge - files: - - policy.yaml +- name: envoy-config + behavior: merge + files: + - policy.yaml + - swagger_template.yaml # TODO(Bobgy): migrate kustomize vars to kpt setters? vars: - name: appName diff --git a/kubeflow/common/iap-ingress/base/swagger_template.yaml b/kubeflow/common/iap-ingress/base/swagger_template.yaml new file mode 100644 index 00000000..c4abb92b --- /dev/null +++ b/kubeflow/common/iap-ingress/base/swagger_template.yaml @@ -0,0 +1,64 @@ +swagger: "2.0" +info: + description: "wildcard config for any HTTP service." + title: "General HTTP Service." + version: "1.0.0" +host: "ENDPOINT_NAME" +x-google-endpoints: +- name: "ENDPOINT_NAME" + target: "INGRESS_TARGET_IP" +basePath: "/" +consumes: +- "application/json" +produces: +- "application/json" +schemes: +- "http" +- "https" +paths: + "/**": + get: + operationId: Get + responses: + '200': + description: Get + default: + description: Error + delete: + operationId: Delete + responses: + '204': + description: Delete + default: + description: Error + patch: + operationId: Patch + responses: + '200': + description: Patch + default: + description: Error + post: + operationId: Post + responses: + '200': + description: Post + default: + description: Error + put: + operationId: Put + responses: + '200': + description: Put + default: + description: Error +security: +- google_jwt: [] +securityDefinitions: + google_jwt: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + x-google-issuer: "https://cloud.google.com/iap" + x-google-jwks_uri: "https://www.gstatic.com/iap/verify/public_key-jwk" + x-google-audiences: "JWT_AUDIENCE" diff --git a/kubeflow/config.yaml b/kubeflow/config.yaml index 9f156a76..3ef53cca 100644 --- a/kubeflow/config.yaml +++ b/kubeflow/config.yaml @@ -55,10 +55,6 @@ components: # https://metacontroller.github.io/metacontroller/ - contrib/metacontroller -# Cloud Endpoints Controller -# https://github.com/danisla/cloud-endpoints-controller -- common/cloud-endpoints - # Resources to deploy a Cloud Endpoint + IAP ingress. # dependencies: [ common/cloud-endpoints ] - common/iap-ingress @@ -115,14 +111,9 @@ components: # dependencies: [ common/cert-manager ] - common/knative -# KFServing -# Migrated from KFServing to KServe since Kubeflow 1.5 -# Don't install both KFServing and KServe, it will cause problems using either one. -# If you want to use KFServing, also checkout the kubeflow/apps/centraldashboard/kustomization.yaml file for instruction. -# dependencies: [ common/knative ] -# - apps/kfserving - # KServe +# We deprecated KFServing in favor of KServe beginning Kubeflow 1.6 +# How to migrate: https://www.kubeflow.org/docs/external-add-ons/kserve/migration/ # dependencies: [ common/knative ] - contrib/kserve diff --git a/kubeflow/pull-upstream.sh b/kubeflow/pull-upstream.sh index 24578608..8a5ba1ba 100755 --- a/kubeflow/pull-upstream.sh +++ b/kubeflow/pull-upstream.sh @@ -16,7 +16,7 @@ set -ex -export KUBEFLOW_MANIFESTS_VERSION=v1.5.1 +export KUBEFLOW_MANIFESTS_VERSION=v1.6.0-rc.1 export KUBEFLOW_MANIFESTS_REPO=https://github.com/kubeflow/manifests.git # Pull Kubeflow Pipelines upstream manifests. @@ -63,13 +63,6 @@ fi mkdir -p apps/training-operator kpt pkg get "${KUBEFLOW_MANIFESTS_REPO}/apps/training-operator/upstream@${KUBEFLOW_MANIFESTS_VERSION}" apps/training-operator -if [ -d apps/kfserving/upstream ]; then - rm -rf apps/kfserving/upstream -fi -mkdir -p apps/kfserving -kpt pkg get "${KUBEFLOW_MANIFESTS_REPO}/apps/kfserving/upstream@${KUBEFLOW_MANIFESTS_VERSION}" apps/kfserving -rm apps/kfserving/upstream/Kptfile - if [ -d apps/katib/upstream ]; then rm -rf apps/katib/upstream fi @@ -109,7 +102,7 @@ if [ -d common/istio/upstream/ ]; then rm -rf common/istio/upstream/ fi mkdir -p common/istio -kpt pkg get "${KUBEFLOW_MANIFESTS_REPO}/common/istio-1-11/@${KUBEFLOW_MANIFESTS_VERSION}" common/istio/upstream/ +kpt pkg get "${KUBEFLOW_MANIFESTS_REPO}/common/istio-1-14/@${KUBEFLOW_MANIFESTS_VERSION}" common/istio/upstream/ rm common/istio/upstream/Kptfile if [ -d common/cert-manager/upstream/ ]; then diff --git a/management/manifests/cnrm-install/README.md b/management/manifests/cnrm-install/README.md index e20816c7..19fdf9cc 100644 --- a/management/manifests/cnrm-install/README.md +++ b/management/manifests/cnrm-install/README.md @@ -1,7 +1,9 @@ # Configuration for installing Cloud Config Connector in the management cluster. -Configs are a copy of the CNRM operator install with necessary Google Cloud -resources to set up workload identity binding. (see [docs](https://cloud.google.com/config-connector/docs/how-to/advanced-install#manual)). +> **Note**: +> Starting with Kubeflow 1.5, we leveraged the managed version of Config Connector, which is called [Config Controller](https://cloud.google.com/anthos-config-management/docs/concepts/config-controller-overview). As it does not require manual upgrading, ignore the upgrade instructions below if you use a default deployment of the management cluster through Config Controller setup process. + +Configs are a copy of the CNRM operator install with necessary Google Cloud resources to set up workload identity binding. (see [docs](https://cloud.google.com/config-connector/docs/how-to/advanced-install#manual)). To update: diff --git a/management/manifests/cnrm-install/install-system/configconnector-operator.yaml b/management/manifests/cnrm-install/install-system/configconnector-operator.yaml index 03af7508..ad67ed96 100644 --- a/management/manifests/cnrm-install/install-system/configconnector-operator.yaml +++ b/management/manifests/cnrm-install/install-system/configconnector-operator.yaml @@ -2,17 +2,17 @@ apiVersion: v1 kind: Namespace metadata: annotations: - cnrm.cloud.google.com/operator-version: 1.46.0 + cnrm.cloud.google.com/operator-version: 1.89.0 labels: cnrm.cloud.google.com/operator-system: "true" name: configconnector-operator-system --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - cnrm.cloud.google.com/operator-version: 1.46.0 - controller-gen.kubebuilder.io/version: v0.2.4 + cnrm.cloud.google.com/operator-version: 1.89.0 + controller-gen.kubebuilder.io/version: v0.6.2 creationTimestamp: null labels: cnrm.cloud.google.com/operator-system: "true" @@ -25,66 +25,85 @@ spec: plural: configconnectorcontexts singular: configconnectorcontext scope: Namespaced - validation: - openAPIV3Schema: - description: ConfigConnectorContext is the Schema for the ConfigConnectorContexts - API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ConfigConnectorContextSpec defines the desired state of ConfigConnectorContext - properties: - googleServiceAccount: - description: The Google Service Account to be used by Config Connector - to authenticate with Google Cloud APIs in the associated namespace. - type: string - requestProjectPolicy: - description: Specifies which project to use for preconditions, quota, - and billing for requests made to Google Cloud APIs for resources in - the associated namespace. Must be one of 'SERVICE_ACCOUNT_PROJECT' - or 'RESOURCE_PROJECT'. Defaults to 'SERVICE_ACCOUNT_PROJECT'. If set - to 'SERVICE_ACCOUNT_PROJECT', uses the project that the Google Service - Account belongs to. If set to 'RESOURCE_PROJECT', uses the project - that the resource belongs to. - enum: - - SERVICE_ACCOUNT_PROJECT - - RESOURCE_PROJECT - type: string - required: - - googleServiceAccount - type: object - status: - description: ConfigConnectorContextStatus defines the observed state of - ConfigConnectorContext - properties: - errors: - items: - type: string - type: array - healthy: - type: boolean - required: - - healthy - type: object - required: - - spec - type: object - version: v1beta1 versions: - - name: v1beta1 + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: When 'true' the most recent reconcile of the ConfigConnectorContext + object succeeded + jsonPath: .status.healthy + name: Healthy + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ConfigConnectorContext is the Schema for the ConfigConnectorContexts + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ConfigConnectorContextSpec defines the desired state of ConfigConnectorContext + properties: + billingProject: + description: Specifies the project to use for preconditions, quota + and billing. Should only be used when requestProjectPolicy is set + to BILLING_PROJECT. + type: string + googleServiceAccount: + description: The Google Service Account to be used by Config Connector + to authenticate with Google Cloud APIs in the associated namespace. + type: string + requestProjectPolicy: + description: Specifies which project to use for preconditions, quota, + and billing for requests made to Google Cloud APIs for resources + in the associated namespace. Must be one of 'SERVICE_ACCOUNT_PROJECT', + 'RESOURCE_PROJECT', or 'BILLING_PROJECT. Defaults to 'SERVICE_ACCOUNT_PROJECT'. + If set to 'SERVICE_ACCOUNT_PROJECT', uses the project that the Google + Service Account belongs to. If set to 'RESOURCE_PROJECT', uses the + project that the resource belongs to. If set to 'BILLING_PROJECT', + uses the project specified by spec.billingProject. + enum: + - SERVICE_ACCOUNT_PROJECT + - RESOURCE_PROJECT + - BILLING_PROJECT + type: string + required: + - googleServiceAccount + type: object + status: + description: ConfigConnectorContextStatus defines the observed state of + ConfigConnectorContext + properties: + errors: + items: + type: string + type: array + healthy: + type: boolean + phase: + type: string + required: + - healthy + type: object + required: + - spec + type: object served: true storage: true + subresources: + status: {} status: acceptedNames: kind: "" @@ -92,12 +111,12 @@ status: conditions: [] storedVersions: [] --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - cnrm.cloud.google.com/operator-version: 1.46.0 - controller-gen.kubebuilder.io/version: v0.2.4 + cnrm.cloud.google.com/operator-version: 1.89.0 + controller-gen.kubebuilder.io/version: v0.6.2 creationTimestamp: null labels: cnrm.cloud.google.com/operator-system: "true" @@ -110,104 +129,118 @@ spec: plural: configconnectors singular: configconnector scope: Cluster - validation: - openAPIV3Schema: - description: ConfigConnector is the Schema for the configconnectors API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - anyOf: - - oneOf: - - not: + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: When 'true' the most recent reconcile of the ConfigConnector object + succeeded + jsonPath: .status.healthy + name: Healthy + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ConfigConnector is the Schema for the configconnectors API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + anyOf: + - oneOf: + - not: + required: + - googleServiceAccount + required: + - credentialSecretName + - not: + required: + - credentialSecretName required: - googleServiceAccount - required: - - credentialSecretName + properties: + mode: + enum: + - cluster - not: - required: - - credentialSecretName - required: - - googleServiceAccount + anyOf: + - required: + - googleServiceAccount + - required: + - credentialSecretName + properties: + mode: + enum: + - namespaced + description: ConfigConnectorSpec defines the desired state of ConfigConnector properties: + credentialSecretName: + description: The Kubernetes secret that contains the Google Service + Account Key's credentials to be used by ConfigConnector to authenticate + with Google Cloud APIs. This field is used only when in cluster + mode. It's recommended to use `googleServiceAccount` when running + ConfigConnector in Google Kubernetes Engine (GKE) clusters with + Workload Identity enabled. This field cannot be specified together + with `googleServiceAccount`. + type: string + googleServiceAccount: + description: The Google Service Account to be used by Config Connector + to authenticate with Google Cloud APIs. This field is used only + when running in cluster mode with Workload Identity enabled. See + Google Kubernetes Engine (GKE) workload-identity (https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) + for details. This field cannot be specified together with `credentialSecretName`. + For namespaced mode, use `googleServiceAccount` in ConfigConnectorContext + CRD to specify the Google Service Account to be used to authenticate + with Google Cloud APIs per namespace. + type: string mode: + description: The mode that Config Connector will run in. This can + be either 'cluster' or 'namespaced'. The default is 'namespaced'. + Cluster mode uses a single Google Service Account to create and + manage resources, even if you are using Config Connector to manage + multiple Projects. You must specify either `credentialSecretName` + or `googleServiceAccount` when in cluster mode, but not both. Namespaced + mode allows you to use different Google service accounts for different + Projects. When in namespaced mode, you must create a ConfigConnectorContext + object per namespace that you want to enable Config Connector in, + and each must set `googleServiceAccount` to specify the Google Service + Account to be used to authenticate with Google Cloud APIs for the + namespace. enum: - cluster - - not: - anyOf: - - required: - - googleServiceAccount - - required: - - credentialSecretName - properties: - mode: - enum: - namespaced - description: ConfigConnectorSpec defines the desired state of ConfigConnector - properties: - credentialSecretName: - description: The Kubernetes secret that contains the Google Service - Account Key's credentials to be used by ConfigConnector to authenticate - with Google Cloud APIs. This field is used only when in cluster mode. - It's recommended to use `googleServiceAccount` when running ConfigConnector - in Google Kubernetes Engine (GKE) clusters with Workload Identity - enabled. This field cannot be specified together with `googleServiceAccount`. - type: string - googleServiceAccount: - description: The Google Service Account to be used by Config Connector - to authenticate with Google Cloud APIs. This field is used only when - running in cluster mode with Workload Identity enabled. See Google - Kubernetes Engine (GKE) workload-identity (https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) - for details. This field cannot be specified together with `credentialSecretName`. - For namespaced mode, use `googleServiceAccount` in ConfigConnectorContext - CRD to specify the Google Service Account to be used to authenticate - with Google Cloud APIs per namespace. - type: string - mode: - description: The mode that Config Connector will run in. This can be - either 'cluster' or 'namespaced'. The default is 'namespaced'. Cluster - mode uses a single Google Service Account to create and manage resources, - even if you are using Config Connector to manage multiple Projects. - You must specify either `credentialSecretName` or `googleServiceAccount` - when in cluster mode, but not both. Namespaced mode allows you to - use different Google service accounts for different Projects. When - in namespaced mode, you must create a ConfigConnectorContext object - per namespace that you want to enable Config Connector in, and each - must set `googleServiceAccount` to specify the Google Service Account - to be used to authenticate with Google Cloud APIs for the namespace. - enum: - - cluster - - namespaced - type: string - type: object - status: - description: ConfigConnectorStatus defines the observed state of ConfigConnector - properties: - errors: - items: type: string - type: array - healthy: - type: boolean - required: - - healthy - type: object - type: object - version: v1beta1 - versions: - - name: v1beta1 + type: object + status: + description: ConfigConnectorStatus defines the observed state of ConfigConnector + properties: + errors: + items: + type: string + type: array + healthy: + type: boolean + phase: + type: string + required: + - healthy + type: object + type: object served: true storage: true + subresources: + status: {} status: acceptedNames: kind: "" @@ -219,7 +252,7 @@ apiVersion: v1 kind: ServiceAccount metadata: annotations: - cnrm.cloud.google.com/operator-version: 1.46.0 + cnrm.cloud.google.com/operator-version: 1.89.0 labels: cnrm.cloud.google.com/operator-system: "true" name: configconnector-operator @@ -229,7 +262,79 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: - cnrm.cloud.google.com/operator-version: 1.46.0 + cnrm.cloud.google.com/operator-version: 1.89.0 + cnrm.cloud.google.com/version: 1.89.0 + creationTimestamp: null + labels: + cnrm.cloud.google.com/operator-system: "true" + cnrm.cloud.google.com/system: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: configconnector-operator-cnrm-viewer +rules: +- apiGroups: + - accesscontextmanager.cnrm.cloud.google.com + - apigee.cnrm.cloud.google.com + - artifactregistry.cnrm.cloud.google.com + - bigquery.cnrm.cloud.google.com + - bigtable.cnrm.cloud.google.com + - billingbudgets.cnrm.cloud.google.com + - binaryauthorization.cnrm.cloud.google.com + - cloudbuild.cnrm.cloud.google.com + - cloudfunctions.cnrm.cloud.google.com + - cloudidentity.cnrm.cloud.google.com + - cloudscheduler.cnrm.cloud.google.com + - compute.cnrm.cloud.google.com + - configcontroller.cnrm.cloud.google.com + - container.cnrm.cloud.google.com + - containeranalysis.cnrm.cloud.google.com + - dataflow.cnrm.cloud.google.com + - datafusion.cnrm.cloud.google.com + - dataproc.cnrm.cloud.google.com + - dns.cnrm.cloud.google.com + - eventarc.cnrm.cloud.google.com + - filestore.cnrm.cloud.google.com + - firestore.cnrm.cloud.google.com + - gameservices.cnrm.cloud.google.com + - gkehub.cnrm.cloud.google.com + - iam.cnrm.cloud.google.com + - iap.cnrm.cloud.google.com + - identityplatform.cnrm.cloud.google.com + - kms.cnrm.cloud.google.com + - logging.cnrm.cloud.google.com + - memcache.cnrm.cloud.google.com + - monitoring.cnrm.cloud.google.com + - networkconnectivity.cnrm.cloud.google.com + - networksecurity.cnrm.cloud.google.com + - networkservices.cnrm.cloud.google.com + - osconfig.cnrm.cloud.google.com + - privateca.cnrm.cloud.google.com + - pubsub.cnrm.cloud.google.com + - recaptchaenterprise.cnrm.cloud.google.com + - redis.cnrm.cloud.google.com + - resourcemanager.cnrm.cloud.google.com + - run.cnrm.cloud.google.com + - secretmanager.cnrm.cloud.google.com + - servicedirectory.cnrm.cloud.google.com + - servicenetworking.cnrm.cloud.google.com + - serviceusage.cnrm.cloud.google.com + - sourcerepo.cnrm.cloud.google.com + - spanner.cnrm.cloud.google.com + - sql.cnrm.cloud.google.com + - storage.cnrm.cloud.google.com + - storagetransfer.cnrm.cloud.google.com + - vpcaccess.cnrm.cloud.google.com + resources: + - '*' + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + cnrm.cloud.google.com/operator-version: 1.89.0 creationTimestamp: null labels: cnrm.cloud.google.com/operator-system: "true" @@ -328,6 +433,7 @@ rules: - core.cnrm.cloud.google.com resources: - configconnectors/status + - configconnectorcontexts/status verbs: - get - patch @@ -388,7 +494,24 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: annotations: - cnrm.cloud.google.com/operator-version: 1.46.0 + cnrm.cloud.google.com/operator-version: 1.89.0 + labels: + cnrm.cloud.google.com/operator-system: "true" + name: configconnector-operator-cnrm-viewer-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: configconnector-operator-cnrm-viewer +subjects: +- kind: ServiceAccount + name: configconnector-operator + namespace: configconnector-operator-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + cnrm.cloud.google.com/operator-version: 1.89.0 labels: cnrm.cloud.google.com/operator-system: "true" name: configconnector-operator-rolebinding @@ -405,7 +528,7 @@ apiVersion: v1 kind: Service metadata: annotations: - cnrm.cloud.google.com/operator-version: 1.46.0 + cnrm.cloud.google.com/operator-version: 1.89.0 labels: cnrm.cloud.google.com/operator-system: "true" name: configconnector-operator-service @@ -422,7 +545,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: annotations: - cnrm.cloud.google.com/operator-version: 1.46.0 + cnrm.cloud.google.com/operator-version: 1.89.0 labels: cnrm.cloud.google.com/component: configconnector-operator cnrm.cloud.google.com/operator-system: "true" @@ -437,7 +560,7 @@ spec: template: metadata: annotations: - cnrm.cloud.google.com/operator-version: 1.46.0 + cnrm.cloud.google.com/operator-version: 1.89.0 labels: cnrm.cloud.google.com/component: configconnector-operator cnrm.cloud.google.com/operator-system: "true" @@ -447,7 +570,8 @@ spec: - --local-repo=/configconnector-operator/channels command: - /configconnector-operator/manager - image: gcr.io/gke-release/cnrm/operator:70a9bee + image: gcr.io/gke-release/cnrm/operator:35d3b61 + imagePullPolicy: Always name: manager resources: limits: