Update KServe component, Kserve guide and fix incorrect paths (#266)

**Description of your changes:** - Kserve offers [2 installation options](https://github.com/kserve/kserve/tree/release-0.7/install/v0.7.0) for serverless: standalone installation (in its own namespace) and a Kubeflow one. In KF-1.5 since both KFServing and KServe are included, Kserve standalone installation was used to avoid conflicts in resource names. But, in doing so the default ingress gateway configured became `kntaive-serving/knative-ingress-gateway` which does not exist in Kubeflow installation. Kubeflow integrated installation should use `kubeflow/kubeflow-gateway`. Overlay under `awsconfig/apps/kserve` fixes this and should eventually be merged upstream - Previous Kserve example fails to load the model because of incompatible libraries in container and hence had to update the sample used in guide - Fixed a bunch of path issues and outdated content and commands
awslabs · Jun 22, 2022 · eebda1c · eebda1c
1 parent b05cc90
commit eebda1c
Show file tree

Hide file tree

Showing 16 changed files with 246 additions and 149 deletions.
diff --git a/awsconfigs/apps/kserve/kustomization.yaml b/awsconfigs/apps/kserve/kustomization.yaml
@@ -0,0 +1,21 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# kserve offers 2 installation options for serverless: standalone installation (in its own namespace) and a Kubeflow one
+# See: https://github.com/kserve/kserve/tree/release-0.7/install/v0.7.0
+# In 1.5 since both KFServing and KServe were included, Kserve standalone installation was used to avoid conflicts in resource names
+# but in doing so the default ingress gateway configured became knative-serving/knative-ingress-gateway which does not exist in Kubeflow installation
+# Kubeflow integrated installation should use kubeflow/kubeflow-gateway
+# this overlay fixes what is achieved by https://github.com/kserve/kserve/blob/release-0.7/config/overlays/kubeflow/params.env#L1 and should eventually be merged upstream
+bases:
+- ../../../upstream/contrib/kserve/kserve
+
+# To make namespace for standalone installation kustomizable,
+# variabalize ingress gateway, webhook service name and
+# kserve namespace in webhook configurations
+configMapGenerator:
+  - name: kserve-config
+    namespace: kserve
+    behavior: merge
+    envs:
+    - params.env
diff --git a/awsconfigs/apps/kserve/params.env b/awsconfigs/apps/kserve/params.env
@@ -0,0 +1 @@
+ingressGateway=kubeflow/kubeflow-gateway
diff --git a/deployments/cognito-rds-s3/kustomization.yaml b/deployments/cognito-rds-s3/kustomization.yaml
@@ -42,7 +42,7 @@ resources:
   - ../../awsconfigs/common/aws-telemetry
 
   # KServe
-  - ../../upstream/contrib/kserve/kserve
+  - ../../awsconfigs/apps/kserve
   - ../../upstream/contrib/kserve/models-web-app/overlays/kubeflow
 
   # Configured for AWS Cognito

diff --git a/deployments/cognito/kustomization.yaml b/deployments/cognito/kustomization.yaml
@@ -46,7 +46,7 @@ resources:
   - ../../awsconfigs/common/aws-telemetry
 
   # KServe
-  - ../../upstream/contrib/kserve/kserve
+  - ../../awsconfigs/apps/kserve
   - ../../upstream/contrib/kserve/models-web-app/overlays/kubeflow
 
   # Configured for AWS Cognito

diff --git a/deployments/rds-s3/base/kustomization.yaml b/deployments/rds-s3/base/kustomization.yaml
@@ -48,7 +48,7 @@ resources:
   - ../../../upstream/common/user-namespace/base
 
   # KServe
-  - ../../../upstream/contrib/kserve/kserve
+  - ../../../awsconfigs/apps/kserve
   - ../../../upstream/contrib/kserve/models-web-app/overlays/kubeflow
 
   # AWS Telemetry - This is an optional component. See usage tracking documentation for more information

diff --git a/deployments/vanilla/kustomization.yaml b/deployments/vanilla/kustomization.yaml
@@ -52,7 +52,7 @@ resources:
   - ../../upstream/common/user-namespace/base
 
   # KServe
-  - ../../upstream/contrib/kserve/kserve
+  - ../../awsconfigs/apps/kserve
   - ../../upstream/contrib/kserve/models-web-app/overlays/kubeflow
 
   # AWS Telemetry - This is an optional component. See usage tracking documentation for more information

diff --git a/tests/e2e/utils/kserve/inference_sample.py b/tests/e2e/utils/kserve/inference_sample.py
@@ -0,0 +1,47 @@
+import requests
+import os
+import json
+
+from e2e.utils.utils import load_json_file
+
+# common vars
+KUBEFLOW_DOMAIN = os.environ.get("KUBEFLOW_DOMAIN", "kubeflow.example.com")
+PROFILE_NAMESPACE = os.environ.get("PROFILE_NAMESPACE", "staging")
+MODEL_NAME = os.environ.get("MODEL_NAME", "sklearn-irisv2")
+AUTH_PROVIDER = os.environ.get("AUTH_PROVIDER", "dex")
+
+URL = f"https://{MODEL_NAME}.{PROFILE_NAMESPACE}.{KUBEFLOW_DOMAIN}/v2/models/{MODEL_NAME}/infer"
+HEADERS = {"Host": f"{MODEL_NAME}.{PROFILE_NAMESPACE}.{KUBEFLOW_DOMAIN}"}
+DASHBOARD_URL = f"https://kubeflow.{KUBEFLOW_DOMAIN}"
+
+data = load_json_file("./utils/kserve/iris-input.json")
+
+response = None
+if AUTH_PROVIDER != "cognito":
+    USERNAME = os.environ.get("USERNAME", "user@example.com")
+    PASSWORD = os.environ.get("PASSWORD", "12341234")
+
+    def session_cookie(host, login, password):
+        session = requests.Session()
+        response = session.get(host)
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+        }
+        data = {"login": login, "password": password}
+        session.post(response.url, headers=headers, data=data)
+        session_cookie = session.cookies.get_dict()["authservice_session"]
+        return session_cookie
+
+    cookie = {"authservice_session": session_cookie(DASHBOARD_URL, USERNAME, PASSWORD)}
+    response = requests.post(URL, headers=HEADERS, json=data, cookies=cookie)
+else:
+    HTTP_HEADER_NAME = os.environ.get("HTTP_HEADER_NAME", "x-api-key")
+    HTTP_HEADER_VALUE = os.environ.get("HTTP_HEADER_VALUE", "token1")
+    HEADERS[HTTP_HEADER_NAME] = HTTP_HEADER_VALUE
+
+    response = requests.post(URL, headers=HEADERS, json=data)
+
+status_code = response.status_code
+print("Status Code", status_code)
+if status_code == 200:
+    print("JSON Response ", json.dumps(response.json(), indent=2))
diff --git a/tests/e2e/utils/kserve/inference_sample_cognito.py b/tests/e2e/utils/kserve/inference_sample_cognito.py
diff --git a/tests/e2e/utils/kserve/inference_sample_dex.py b/tests/e2e/utils/kserve/inference_sample_dex.py
diff --git a/tests/e2e/utils/kserve/iris-input.json b/tests/e2e/utils/kserve/iris-input.json
@@ -0,0 +1,13 @@
+{
+  "inputs":[
+    {
+      "name":"input-0",
+      "shape":[2, 4],
+      "datatype":"FP32",
+      "data":[
+        [6.8, 2.8, 4.8, 1.4],
+        [6.0, 3.4, 4.5, 1.6]
+      ]
+    }
+  ]
+}
diff --git a/website/content/en/docs/component-guides/kserve.md b/website/content/en/docs/component-guides/kserve.md
@@ -59,14 +59,15 @@ DNS only supports wildcard placeholders in the [leftmost part of the domain name
 
 ### Create a certificate
 > Note: Both of these domains should be requested in the same certificate
-Create an ACM certificate for `*.platform.example.com` and `*.staging.platform.example.com` in your cluster's region by following the [create certificates for domain](/kubeflow-manifests/docs/deployment/add-ons/load-balancer/guide/#create-domain-and-certificates) steps in the Load Balancer installation guide. 
+
+Create an ACM certificate for `*.platform.example.com` and `*.staging.platform.example.com` in your cluster's region by following the [create certificates for domain](/kubeflow-manifests/deployments/add-ons/load-balancer/guide/#create-certificates-for-domain) steps in the Load Balancer installation guide. 
 
 Once the certificate status changes to `Issued`, export the ARN of the certificate created:
 ```bash
 export certArn=<>
 ```
 
-If you are using Cognito for user authentication, see [Cognito](/kubeflow-manifests/docs/component-guides/kserve/#cognito). If you use Dex as the auth provider in your Kubeflow deployment, see [Dex](/kubeflow-manifests/docs/component-guides/kserve/#dex). 
+If you are using Cognito for user authentication, see [Cognito](/kubeflow-manifests/docs/component-guides/kserve/#cognito-ingress). If you use Dex as the auth provider in your Kubeflow deployment, see [Dex](/kubeflow-manifests/docs/component-guides/kserve/#dex-ingress). 
 
 ## Cognito ingress
 
@@ -99,8 +100,8 @@ Use an ingress to set the [HTTP header conditions](https://docs.aws.amazon.com/e
 4. Check if the ingress-managed Load Balancer is provisioned. This may take a few minutes to complete.
     ```bash
     kubectl get ingress -n istio-system istio-ingress-api
-    NAME                CLASS    HOSTS   ADDRESS                                                                PORTS   AGE
-    istio-ingress-api   <none>   *       xxxxxx-istiosystem-istio-2af2-1100502020.us-west-2.elb.amazonaws.com   80      14m
+    NAME                CLASS    HOSTS   ADDRESS                                                              PORTS   AGE
+    istio-ingress-api   <none>   *       k8s-istiosys-istioing-xxxxxx-110050202.us-west-2.elb.amazonaws.com   80      14m
     ```
 
 Once your Load Balancer is ready, move on to the [Add DNS records](/kubeflow-manifests/docs/component-guides/kserve/#add-dns-records) step to add a DNS record for the staging subdomain.
@@ -118,10 +119,10 @@ Once your Load Balancer is ready, move on to the [Add DNS records](/kubeflow-man
     kustomize build awsconfigs/common/istio-ingress/overlays/https | kubectl apply -f -
     ``` 
 3. Get the Load Balancer address
-    ```
+    ```bash
     kubectl get ingress -n istio-system istio-ingress
-    NAME            CLASS    HOSTS   ADDRESS                                                                  PORTS   AGE
-    istio-ingress   <none>   *       xxxxxx-istiosystem-istio-2af2-1100502020.us-west-2.elb.amazonaws.com   80      15d
+    NAME            CLASS    HOSTS   ADDRESS                                                              PORTS   AGE
+    istio-ingress   <none>   *       k8s-istiosys-istioing-xxxxxx-110050202.us-west-2.elb.amazonaws.com   80      15d
     ```
 Once your Load Balancer is ready, move on to the [Add DNS records](/kubeflow-manifests/docs/component-guides/kserve/#add-dns-records) step to add a DNS record for the staging subdomain.
 
@@ -143,60 +144,114 @@ kubectl get authorizationpolicies -n istio-system
 ```
 
 ### Create an `InferenceService`
-Create a scikit-learn `InferenceService` using a [sample](https://github.com/kubeflow/kfserving-lts/blob/release-0.6/docs/samples/v1beta1/sklearn/v1/sklearn.yaml) from the KFserving repository and wait for `READY` to be `True`.
+
+Set the environment variable value for `PROFILE_NAMESPACE`(e.g. `staging`) according to your environment:
+```bash
+export PROFILE_NAMESPACE="staging"
+```
+
+Create a scikit-learn `InferenceService` using a [sample](https://github.com/kserve/kserve/blob/release-0.7/docs/samples/v1beta1/sklearn/v2/sklearn.yaml) from the KFserving repository and wait for `READY` to be `True`.
+
 ```bash
-kubectl apply -n staging -f https://raw.githubusercontent.com/kubeflow/kfserving-lts/release-0.6/docs/samples/v1beta1/sklearn/v1/sklearn.yaml
+kubectl apply -n ${PROFILE_NAMESPACE} -f https://raw.githubusercontent.com/kserve/kserve/release-0.7/docs/samples/v1beta1/sklearn/v2/sklearn.yaml
 ```
 
 ### Check `InferenceService` status
 
 Check the `InferenceService` status. Once it is ready, copy the URL to use for sending a prediction request.
 ```bash
-kubectl get inferenceservices sklearn-iris -n staging
+kubectl get inferenceservices sklearn-irisv2 -n ${PROFILE_NAMESPACE}
 
-NAME           URL                                                READY   PREV   LATEST   PREVROLLEDOUTREVISION   LATESTREADYREVISION                    AGE
-sklearn-iris   http://sklearn-iris.staging.platform.example.com   True           100                              sklearn-iris-predictor-default-00001   3m31s
+NAME             URL                                                 READY   PREV   LATEST   PREVROLLEDOUTREVISION   LATESTREADYREVISION                      AGE
+sklearn-irisv2   http://sklearn-iris2.staging.platform.example.com   True           100                              sklearn-irisv2-predictor-default-00001   3m31s
 ```
 
 ### Send an inference request
 
-Set the environment variable values for `KUBEFLOW_DOMAIN`(e.g. `platform.example.com`) and `PROFILE_NAMESPACE`(e.g. `staging`) according to your environment:
-```
+Set the environment variable values for `KUBEFLOW_DOMAIN`(e.g. `platform.example.com`) according to your environment:
+```bash
 export KUBEFLOW_DOMAIN="platform.example.com"
-export PROFILE_NAMESPACE="staging"
 ```
 
-Install dependencies for the script by running `pip install requests`
+Install dependencies for the script by running:
+```bash
+cd tests/e2e
+pip install requirements.txt
+```
 
 Run the sample python script to send an inference request based on your auth provider:
 
 #### Cognito inference 
 
-Run the [inference_sample_cognito.py](https://github.com/awslabs/kubeflow-manifests/blob/main/tests/e2e/utils/kserve/inference_sample_cognito.py) Python script by exporting the values for `HTTP_HEADER_NAME`(e.g. `x-api-key`) and `HTTP_HEADER_VALUE`(e.g. `token1`) according to the values configured in [ingress section](/kubeflow-manifests/docs/component-guides/kserve/#create-ingress).
+Run the [inference_sample.py](https://github.com/awslabs/kubeflow-manifests/blob/main/tests/e2e/utils/kserve/inference_sample.py) Python script by exporting the values for `HTTP_HEADER_NAME`(e.g. `x-api-key`) and `HTTP_HEADER_VALUE`(e.g. `token1`) according to the values configured in [ingress section](/kubeflow-manifests/docs/component-guides/kserve/#create-ingress).
 ```bash
+export AUTH_PROVIDER="cognito"
 export HTTP_HEADER_NAME="x-api-key"
 export HTTP_HEADER_VALUE="token1"
+```
 
-python inference_sample_cognito.py
+```bash
+PYTHONPATH=.. python utils/kserve/inference_sample.py
 ```
 
 The output should look similar to the following:
 ```bash
 Status Code 200
-JSON Response  {'predictions': [1, 1]}
+JSON Response  {
+  "model_name": "sklearn-irisv2",
+  "model_version": null,
+  "id": "e5fc40ba-5f02-42f7-aff8-34042facbe11",
+  "parameters": null,
+  "outputs": [
+    {
+      "name": "predict",
+      "shape": [
+        2
+      ],
+      "datatype": "FP32",
+      "parameters": null,
+      "data": [
+        1,
+        2
+      ]
+    }
+  ]
+}
 ```
 
 #### Dex inference
-Run the [inference_sample_dex.py](https://github.com/awslabs/kubeflow-manifests/blob/main/tests/e2e/utils/kserve/inference_sample_dex.py) Python script by exporting the values for `USERNAME`(e.g. `user@example.com`), `PASSWORD` according to the user profile 
+Run the [inference_sample.py](https://github.com/awslabs/kubeflow-manifests/blob/main/tests/e2e/utils/kserve/inference_sample.py) Python script by exporting the values for `USERNAME`(e.g. `user@example.com`), `PASSWORD` according to the user profile 
 ```bash
+export AUTH_PROVIDER="dex"
 export USERNAME="user@example.com"
 export PASSWORD="12341234"
+```
 
-python inference_sample_dex.py
+```bash
+PYTHONPATH=.. python utils/kserve/inference_sample.py
 ```
 
 The output should look similar to the following:
 ```bash
 Status Code 200
-JSON Response  {'predictions': [1, 1]}
+JSON Response  {
+  "model_name": "sklearn-irisv2",
+  "model_version": null,
+  "id": "e5fc40ba-5f02-42f7-aff8-34042facbe11",
+  "parameters": null,
+  "outputs": [
+    {
+      "name": "predict",
+      "shape": [
+        2
+      ],
+      "datatype": "FP32",
+      "parameters": null,
+      "data": [
+        1,
+        2
+      ]
+    }
+  ]
+}
 ```
diff --git a/website/content/en/docs/deployment/add-ons/load-balancer/guide.md b/website/content/en/docs/deployment/add-ons/load-balancer/guide.md
@@ -129,8 +129,8 @@ while ! kustomize build deployments/add-ons/load-balancer | kubectl apply -f -;
 1. Check if ALB is provisioned. This may take a few minutes.
     ```
         kubectl get ingress -n istio-system istio-ingress
-        NAME            CLASS    HOSTS   ADDRESS                                                                  PORTS   AGE
-        istio-ingress   <none>   *       xxxxxx-istiosystem-istio-2af2-1100502020.us-west-2.elb.amazonaws.com   80      15d
+        NAME            CLASS    HOSTS   ADDRESS                                                              PORTS   AGE
+        istio-ingress   <none>   *       k8s-istiosys-istioing-xxxxxx-110050202.us-west-2.elb.amazonaws.com   80      15d
     ```
     If `ADDRESS` is empty after a few minutes, check the logs of the controller by following the troubleshooting steps in [ALB fails to provision](https://awslabs.github.io/kubeflow-manifests/docs/troubleshooting-aws/#alb-fails-to-provision).
 2. When ALB is ready, copy the DNS name of that load balancer and create a CNAME entry to it in Route53 under the subdomain (`platform.example.com`) for `*.platform.example.com`. Please note that it might make up to five to ten minutes for DNS changes to propagate and for your URL to work.
@@ -189,6 +189,7 @@ while ! kustomize build deployments/add-ons/load-balancer | kubectl apply -f -;
             name: platform.example.com
     ```
 1. The central dashboard should now be available at `https://kubeflow.platform.example.com`. Open a browser and navigate to this URL.
+> Note: It might a few minutes for DNS changes to propagate and for your URL to work. Check if the DNS entry propogated with the [Google Admin Toolbox](https://toolbox.googleapps.com/apps/dig/#CNAME/)
 
 ## Clean up
 

diff --git a/website/content/en/docs/deployment/cognito-rds-s3/guide.md b/website/content/en/docs/deployment/cognito-rds-s3/guide.md
@@ -54,11 +54,15 @@ Refer to the [general prerequisites guide](/kubeflow-manifests/docs/deployment/p
         # Kubeflow Istio Resources
         kustomize build upstream/common/istio-1-9/kubeflow-istio-resources/base | kubectl apply -f -
         
-        # KFServing
+        # KServe
+        kustomize build awsconfigs/apps/kserve | kubectl apply -f -
+        kustomize build upstream/contrib/kserve/models-web-app/overlays/kubeflow | kubectl apply -f -
+
+        # KFServing -  This is an optional component and required only if you are not ready to migrate to KServe. We recommend migrating to KServe as soon as possible
         kustomize build upstream/apps/kfserving/upstream/overlays/kubeflow | kubectl apply -f -
         
         # Central Dashboard
-        kustomize build upstream/apps/centraldashboard/upstream/overlays/istio | kubectl apply -f -
+        kustomize build upstream/apps/centraldashboard/upstream/overlays/kserve | kubectl apply -f -
         
         # Notebooks
         kustomize build upstream/apps/jupyter/notebook-controller/upstream/overlays/kubeflow | kubectl apply -f -
@@ -76,9 +80,6 @@ Refer to the [general prerequisites guide](/kubeflow-manifests/docs/deployment/p
         # Tensorboard
         kustomize build upstream/apps/tensorboard/tensorboards-web-app/upstream/overlays/istio | kubectl apply -f -
         kustomize build upstream/apps/tensorboard/tensorboard-controller/upstream/overlays/kubeflow | kubectl apply -f -
-        
-        # MPI Operator
-        kustomize build upstream/apps/mpi-job/upstream/overlays/kubeflow | kubectl apply -f -
 
         # Training Operator
         kustomize build upstream/apps/training-operator/upstream/overlays/kubeflow | kubectl apply -f -