diff --git a/doc/source/examples/argo_workflows_hdfs_batch.html b/doc/source/examples/argo_workflows_hdfs_batch.html new file mode 100644 index 0000000000..bd3073b078 --- /dev/null +++ b/doc/source/examples/argo_workflows_hdfs_batch.html @@ -0,0 +1,3 @@ +{ + "path": "../../../examples/batch/hdfs-argo-workflows/hdfs-batch.ipynb" +} diff --git a/doc/source/examples/notebooks.rst b/doc/source/examples/notebooks.rst index c13baeb401..705f1ac596 100644 --- a/doc/source/examples/notebooks.rst +++ b/doc/source/examples/notebooks.rst @@ -92,7 +92,8 @@ Batch Processing with Seldon Core .. toctree:: :titlesonly: - Batch Processing with Argo Workflows + Batch Processing with Argo Workflows and S3 / Minio + Batch Processing with Argo Workflows and HDFS Batch Processing with Kubeflow Pipelines diff --git a/doc/source/servers/batch.md b/doc/source/servers/batch.md index f10af67d4d..be3dcc85a1 100644 --- a/doc/source/servers/batch.md +++ b/doc/source/servers/batch.md @@ -26,7 +26,8 @@ This allows you to leverage Seldon on a large number of batch applications, incl We have provided a set of examples that show you how you can use the Seldon batch processing component: -* [Batch Processing with Argo Workflows](../examples/argo_workflows_batch.html) +* [Batch Processing with Argo Workflows and S3 / Minio](../examples/argo_workflows_batch.html) +* [Batch processing with Argo Workflows and HDFS](../examples/argo_workflows_hdfs_batch.html) * [Batch Processing with Kubeflow Pipelines Example](../examples/kubeflow_pipelines_batch.html) ## High Level Implementation Details diff --git a/examples/batch/argo-workflows-batch/README.ipynb b/examples/batch/argo-workflows-batch/README.ipynb index c538d93cfe..3abad2a63c 100644 --- a/examples/batch/argo-workflows-batch/README.ipynb +++ b/examples/batch/argo-workflows-batch/README.ipynb @@ -12,7 +12,7 @@ "\n", "* Seldon core installed as per the docs with an ingress\n", "* Minio running in your cluster to use as local (s3) object storage\n", - "* Argo Workfklows installed in cluster (and argo CLI for commands)\n" + "* Argo Workfklows installed in cluster (and argo CLI for commands)" ] }, { @@ -29,11 +29,40 @@ "#### Set up Minio in your cluster\n", "Use the notebook to [set-up Minio in your cluster](https://docs.seldon.io/projects/seldon-core/en/latest/examples/minio_setup.html).\n", "\n", - "#### Copy the Minio Secret to namespace\n", - "\n", - "We need to re-use the minio secret for the batch job, so this can be done by just copying the minio secret created in the `minio-system`\n", - "\n", - "The command below just copies the secred with the name \"minio\" from the minio-system namespace to the default namespace." + "#### Create rclone configuration\n", + "In this example, our workflow stages responsible for pulling / pushing data to in-cluster MinIO S3 storage will use `rclone` CLI.\n", + "In order to configure the CLI we will create a following secret:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting rclone-config.yaml\n" + ] + } + ], + "source": [ + "%%writefile rclone-config.yaml\n", + "apiVersion: v1\n", + "kind: Secret\n", + "metadata:\n", + " name: rclone-config-secret\n", + "type: Opaque\n", + "stringData:\n", + " rclone.conf: |\n", + " [cluster-minio]\n", + " type = s3\n", + " provider = minio\n", + " env_auth = false\n", + " access_key_id = minioadmin\n", + " secret_access_key = minioadmin\n", + " endpoint = http://minio.minio-system.svc.cluster.local:9000" ] }, { @@ -45,12 +74,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "secret/minio created\r\n" + "secret/rclone-config-secret created\n" ] } ], "source": [ - "!kubectl get secret minio -n minio-system -o json | jq '{apiVersion,data,kind,metadata,type} | .metadata |= {\"annotations\", \"name\"}' | kubectl apply -n default -f -" + "!kubectl apply -n default -f rclone-config.yaml" ] }, { @@ -65,7 +94,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -111,23 +140,22 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "['Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply',\n", - " 'role.rbac.authorization.k8s.io/workflow configured']" + "['role.rbac.authorization.k8s.io/workflow created']" ] }, - "execution_count": 21, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "!!kubectl apply -f role.yaml" + "!!kubectl apply -n default -f role.yaml" ] }, { @@ -139,19 +167,19 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Error from server (AlreadyExists): serviceaccounts \"workflow\" already exists\r\n" + "serviceaccount/workflow created\n" ] } ], "source": [ - "!kubectl create serviceaccount workflow" + "!kubectl create -n default serviceaccount workflow" ] }, { @@ -163,19 +191,19 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "rolebinding.rbac.authorization.k8s.io/workflow created\r\n" + "rolebinding.rbac.authorization.k8s.io/workflow created\n" ] } ], "source": [ - "!kubectl create rolebinding workflow --role=workflow --serviceaccount=default:workflow" + "!kubectl create rolebinding workflow -n default --role=workflow --serviceaccount=default:workflow" ] }, { @@ -189,7 +217,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -198,13 +226,18 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ + "import random\n", + "import os\n", + "random.seed(0)\n", "with open(\"assets/input-data.txt\", \"w\") as f:\n", - " for i in range(10000):\n", - " f.write('[[1, 2, 3, 4]]\\n')" + " for _ in range(10000):\n", + " data = [random.random() for _ in range(4)]\n", + " data = \"[[\" + \", \".join(str(x) for x in data) + \"]]\\n\"\n", + " f.write(data)" ] }, { @@ -216,7 +249,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -224,16 +257,16 @@ "output_type": "stream", "text": [ "10000 assets/input-data.txt\n", - "[[1, 2, 3, 4]]\n", - "[[1, 2, 3, 4]]\n", - "[[1, 2, 3, 4]]\n", - "[[1, 2, 3, 4]]\n", - "[[1, 2, 3, 4]]\n", - "[[1, 2, 3, 4]]\n", - "[[1, 2, 3, 4]]\n", - "[[1, 2, 3, 4]]\n", - "[[1, 2, 3, 4]]\n", - "[[1, 2, 3, 4]]\n" + "[[0.8444218515250481, 0.7579544029403025, 0.420571580830845, 0.25891675029296335]]\n", + "[[0.5112747213686085, 0.4049341374504143, 0.7837985890347726, 0.30331272607892745]]\n", + "[[0.4765969541523558, 0.5833820394550312, 0.9081128851953352, 0.5046868558173903]]\n", + "[[0.28183784439970383, 0.7558042041572239, 0.6183689966753316, 0.25050634136244054]]\n", + "[[0.9097462559682401, 0.9827854760376531, 0.8102172359965896, 0.9021659504395827]]\n", + "[[0.3101475693193326, 0.7298317482601286, 0.8988382879679935, 0.6839839319154413]]\n", + "[[0.47214271545271336, 0.1007012080683658, 0.4341718354537837, 0.6108869734438016]]\n", + "[[0.9130110532378982, 0.9666063677707588, 0.47700977655271704, 0.8653099277716401]]\n", + "[[0.2604923103919594, 0.8050278270130223, 0.5486993038355893, 0.014041700164018955]]\n", + "[[0.7197046864039541, 0.39882354222426875, 0.824844977148233, 0.6681532012318508]]\n" ] } ], @@ -251,7 +284,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -259,7 +292,7 @@ "output_type": "stream", "text": [ "\u001b[m\u001b[32;1mBucket created successfully `minio-seldon/data`.\u001b[0m\n", - "...-data.txt: 146.48 KiB / 146.48 KiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 14.57 MiB/s 0s\u001b[0m\u001b[0m\u001b[m\u001b[32;1m" + "...-data.txt: 820.96 KiB / 820.96 KiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 71.44 MiB/s 0s\u001b[0m\u001b[0m\u001b[m\u001b[32;1m" ] } ], @@ -283,18 +316,19 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Name: seldon-batch-process\r\n", - "Namespace: default\r\n", - "ServiceAccount: workflow\r\n", - "Status: Pending\r\n", - "Created: Thu Nov 19 15:09:43 +0000 (now)\r\n" + "Name: seldon-batch-process\n", + "Namespace: default\n", + "ServiceAccount: workflow\n", + "Status: Pending\n", + "Created: Fri Jan 15 11:44:56 +0000 (now)\n", + "Progress: \n" ] } ], @@ -313,97 +347,195 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "NAME STATUS AGE DURATION PRIORITY\r\n", - "seldon-batch-process Running 1m 1m 0\r\n" + "NAME STATUS AGE DURATION PRIORITY\n", + "seldon-batch-process Running 10s 10s 0\n" ] } ], "source": [ - "!argo list" + "!argo list -n default" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Name: seldon-batch-process\r\n", - "Namespace: default\r\n", - "ServiceAccount: workflow\r\n", - "Status: Succeeded\r\n", - "Conditions: \r\n", - " Completed True\r\n", - "Created: Thu Nov 19 14:20:58 +0000 (27 minutes ago)\r\n", - "Started: Thu Nov 19 14:20:58 +0000 (27 minutes ago)\r\n", - "Finished: Thu Nov 19 14:25:30 +0000 (23 minutes ago)\r\n", - "Duration: 4 minutes 32 seconds\r\n", - "ResourcesDuration: 7m52s*(1 cpu),7m52s*(100Mi memory)\r\n", - "\r\n", - "\u001b[39mSTEP\u001b[0m TEMPLATE PODNAME DURATION MESSAGE\r\n", - " \u001b[32m✔\u001b[0m seldon-batch-process seldon-batch-process \r\n", - " ├---\u001b[32m✔\u001b[0m create-seldon-resource create-seldon-resource-template seldon-batch-process-3626514072 1s \r\n", - " ├---\u001b[32m✔\u001b[0m wait-seldon-resource wait-seldon-resource-template seldon-batch-process-2052519094 38s \r\n", - " ├---\u001b[32m✔\u001b[0m download-object-store download-object-store-template seldon-batch-process-1257652469 8s \r\n", - " ├---\u001b[32m✔\u001b[0m process-batch-inputs process-batch-inputs-template seldon-batch-process-2033515954 3m \r\n", - " └---\u001b[32m✔\u001b[0m upload-object-store upload-object-store-template seldon-batch-process-2123074048 3s \r\n" + "Name: seldon-batch-process\n", + "Namespace: default\n", + "ServiceAccount: workflow\n", + "Status: Succeeded\n", + "Conditions: \n", + " Completed True\n", + "Created: Fri Jan 15 11:44:56 +0000 (2 minutes ago)\n", + "Started: Fri Jan 15 11:44:56 +0000 (2 minutes ago)\n", + "Finished: Fri Jan 15 11:47:00 +0000 (36 seconds ago)\n", + "Duration: 2 minutes 4 seconds\n", + "Progress: 6/6\n", + "ResourcesDuration: 2m18s*(1 cpu),2m18s*(100Mi memory)\n", + "\n", + "\u001b[39mSTEP\u001b[0m TEMPLATE PODNAME DURATION MESSAGE\n", + " \u001b[32m✔\u001b[0m seldon-batch-process seldon-batch-process \n", + " ├───\u001b[32m✔\u001b[0m create-seldon-resource create-seldon-resource-template seldon-batch-process-3626514072 2s \n", + " ├───\u001b[32m✔\u001b[0m wait-seldon-resource wait-seldon-resource-template seldon-batch-process-2052519094 31s \n", + " ├───\u001b[32m✔\u001b[0m download-object-store download-object-store-template seldon-batch-process-1257652469 4s \n", + " ├───\u001b[32m✔\u001b[0m process-batch-inputs process-batch-inputs-template seldon-batch-process-2033515954 33s \n", + " ├───\u001b[32m✔\u001b[0m upload-object-store upload-object-store-template seldon-batch-process-2123074048 3s \n", + " └───\u001b[32m✔\u001b[0m delete-seldon-resource delete-seldon-resource-template seldon-batch-process-2070809024 9s \n" ] } ], "source": [ - "!argo get seldon-batch-process" + "!argo get -n default seldon-batch-process" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32mseldon-batch-process-3626514072: time=\"2020-11-19T14:20:59.511Z\" level=info msg=\"Starting Workflow Executor\" version=v2.11.7\u001b[0m\r\n", - "\u001b[32mseldon-batch-process-3626514072: time=\"2020-11-19T14:20:59.514Z\" level=info msg=\"Creating a K8sAPI executor\"\u001b[0m\r\n", - "\u001b[32mseldon-batch-process-3626514072: time=\"2020-11-19T14:20:59.514Z\" level=info msg=\"Executor (version: v2.11.7, build_date: 2020-11-02T21:05:12Z) initialized (pod: default/seldon-batch-process-3626514072) with template:\\n{\\\"name\\\":\\\"create-seldon-resource-template\\\",\\\"arguments\\\":{},\\\"inputs\\\":{},\\\"outputs\\\":{},\\\"metadata\\\":{},\\\"resource\\\":{\\\"action\\\":\\\"create\\\",\\\"manifest\\\":\\\"apiVersion: machinelearning.seldon.io/v1\\\\nkind: SeldonDeployment\\\\nmetadata:\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n namespace: default\\\\n ownerReferences:\\\\n - apiVersion: argoproj.io/v1alpha1\\\\n blockOwnerDeletion: true\\\\n kind: Workflow\\\\n name: \\\\\\\"seldon-batch-process\\\\\\\"\\\\n uid: \\\\\\\"3dc52b6d-937c-47a8-b5f7-d3ca99c74f4e\\\\\\\"\\\\nspec:\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n predictors:\\\\n - componentSpecs:\\\\n - spec:\\\\n containers:\\\\n - name: classifier\\\\n env:\\\\n - name: GUNICORN_THREADS\\\\n value: 10\\\\n - name: GUNICORN_WORKERS\\\\n value: 1\\\\n resources:\\\\n requests:\\\\n cpu: 50m\\\\n memory: 100Mi\\\\n limits:\\\\n cpu: 50m\\\\n memory: 1000Mi\\\\n graph:\\\\n children: []\\\\n implementation: SKLEARN_SERVER\\\\n modelUri: gs://seldon-models/sklearn/iris\\\\n name: classifier\\\\n name: default\\\\n replicas: 10\\\\n \\\\n\\\"}}\"\u001b[0m\r\n", - "\u001b[32mseldon-batch-process-3626514072: time=\"2020-11-19T14:20:59.514Z\" level=info msg=\"Loading manifest to /tmp/manifest.yaml\"\u001b[0m\r\n", - "\u001b[32mseldon-batch-process-3626514072: time=\"2020-11-19T14:20:59.514Z\" level=info msg=\"kubectl create -f /tmp/manifest.yaml -o json\"\u001b[0m\r\n", - "\u001b[32mseldon-batch-process-3626514072: time=\"2020-11-19T14:20:59.799Z\" level=info msg=default/SeldonDeployment.machinelearning.seldon.io/sklearn\u001b[0m\r\n", - "\u001b[32mseldon-batch-process-3626514072: time=\"2020-11-19T14:20:59.799Z\" level=info msg=\"Starting SIGUSR2 signal monitor\"\u001b[0m\r\n", - "\u001b[32mseldon-batch-process-3626514072: time=\"2020-11-19T14:20:59.799Z\" level=info msg=\"No output parameters\"\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 0 of 10 updated replicas are available...\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 1 of 10 updated replicas are available...\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 2 of 10 updated replicas are available...\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 3 of 10 updated replicas are available...\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 4 of 10 updated replicas are available...\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 5 of 10 updated replicas are available...\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 6 of 10 updated replicas are available...\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 7 of 10 updated replicas are available...\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 8 of 10 updated replicas are available...\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 9 of 10 updated replicas are available...\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2052519094: deployment \"sklearn-default-0-classifier\" successfully rolled out\u001b[0m\r\n", - "\u001b[36mseldon-batch-process-1257652469: Added `minio-local` successfully.\u001b[0m\r\n", - "\u001b[36mseldon-batch-process-1257652469: `minio-local/data/input-data.txt` -> `/assets/input-data.txt`\u001b[0m\r\n", - "\u001b[36mseldon-batch-process-1257652469: Total: 0 B, Transferred: 146.48 KiB, Speed: 9.37 MiB/s\u001b[0m\r\n", - "\u001b[33mseldon-batch-process-2033515954: Elapsed time: 181.3895139694214\u001b[0m\r\n", - "\u001b[34mseldon-batch-process-2123074048: Added `minio-local` successfully.\u001b[0m\r\n", - "\u001b[34mseldon-batch-process-2123074048: `/assets/output-data.txt` -> `minio-local/data/output-data-3dc52b6d-937c-47a8-b5f7-d3ca99c74f4e.txt`\u001b[0m\r\n", - "\u001b[34mseldon-batch-process-2123074048: Total: 0 B, Transferred: 5.17 MiB, Speed: 83.04 MiB/s\u001b[0m\r\n" + "\u001b[32mseldon-batch-process-3626514072: time=\"2021-01-15T11:44:57.620Z\" level=info msg=\"Starting Workflow Executor\" version=v2.12.3\u001b[0m\n", + "\u001b[32mseldon-batch-process-3626514072: time=\"2021-01-15T11:44:57.622Z\" level=info msg=\"Creating a K8sAPI executor\"\u001b[0m\n", + "\u001b[32mseldon-batch-process-3626514072: time=\"2021-01-15T11:44:57.622Z\" level=info msg=\"Executor (version: v2.12.3, build_date: 2021-01-05T00:54:54Z) initialized (pod: default/seldon-batch-process-3626514072) with template:\\n{\\\"name\\\":\\\"create-seldon-resource-template\\\",\\\"arguments\\\":{},\\\"inputs\\\":{},\\\"outputs\\\":{},\\\"metadata\\\":{\\\"annotations\\\":{\\\"sidecar.istio.io/inject\\\":\\\"false\\\"}},\\\"resource\\\":{\\\"action\\\":\\\"create\\\",\\\"manifest\\\":\\\"apiVersion: machinelearning.seldon.io/v1\\\\nkind: SeldonDeployment\\\\nmetadata:\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n namespace: default\\\\n ownerReferences:\\\\n - apiVersion: argoproj.io/v1alpha1\\\\n blockOwnerDeletion: true\\\\n kind: Workflow\\\\n name: \\\\\\\"seldon-batch-process\\\\\\\"\\\\n uid: \\\\\\\"511f64a2-0699-42eb-897a-c0a57b24072c\\\\\\\"\\\\nspec:\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n predictors:\\\\n - componentSpecs:\\\\n - spec:\\\\n containers:\\\\n - name: classifier\\\\n env:\\\\n - name: GUNICORN_THREADS\\\\n value: 10\\\\n - name: GUNICORN_WORKERS\\\\n value: 1\\\\n resources:\\\\n requests:\\\\n cpu: 50m\\\\n memory: 100Mi\\\\n limits:\\\\n cpu: 50m\\\\n memory: 1000Mi\\\\n graph:\\\\n children: []\\\\n implementation: SKLEARN_SERVER\\\\n modelUri: gs://seldon-models/sklearn/iris\\\\n name: classifier\\\\n name: default\\\\n replicas: 10\\\\n\\\"}}\"\u001b[0m\n", + "\u001b[32mseldon-batch-process-3626514072: time=\"2021-01-15T11:44:57.622Z\" level=info msg=\"Loading manifest to /tmp/manifest.yaml\"\u001b[0m\n", + "\u001b[32mseldon-batch-process-3626514072: time=\"2021-01-15T11:44:57.622Z\" level=info msg=\"kubectl create -f /tmp/manifest.yaml -o json\"\u001b[0m\n", + "\u001b[32mseldon-batch-process-3626514072: time=\"2021-01-15T11:44:58.044Z\" level=info msg=default/SeldonDeployment.machinelearning.seldon.io/sklearn\u001b[0m\n", + "\u001b[32mseldon-batch-process-3626514072: time=\"2021-01-15T11:44:58.044Z\" level=info msg=\"Starting SIGUSR2 signal monitor\"\u001b[0m\n", + "\u001b[32mseldon-batch-process-3626514072: time=\"2021-01-15T11:44:58.045Z\" level=info msg=\"No output parameters\"\u001b[0m\n", + "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 0 of 10 updated replicas are available...\u001b[0m\n", + "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 1 of 10 updated replicas are available...\u001b[0m\n", + "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 2 of 10 updated replicas are available...\u001b[0m\n", + "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 3 of 10 updated replicas are available...\u001b[0m\n", + "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 4 of 10 updated replicas are available...\u001b[0m\n", + "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 5 of 10 updated replicas are available...\u001b[0m\n", + "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 6 of 10 updated replicas are available...\u001b[0m\n", + "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 7 of 10 updated replicas are available...\u001b[0m\n", + "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 8 of 10 updated replicas are available...\u001b[0m\n", + "\u001b[33mseldon-batch-process-2052519094: Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 9 of 10 updated replicas are available...\u001b[0m\n", + "\u001b[33mseldon-batch-process-2052519094: deployment \"sklearn-default-0-classifier\" successfully rolled out\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:00,306 - batch_processor.py:167 - INFO: Processed instances: 100\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:00,321 - batch_processor.py:167 - INFO: Processed instances: 200\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:00,411 - batch_processor.py:167 - INFO: Processed instances: 300\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:00,464 - batch_processor.py:167 - INFO: Processed instances: 400\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:00,768 - batch_processor.py:167 - INFO: Processed instances: 500\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:01,055 - batch_processor.py:167 - INFO: Processed instances: 600\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:01,439 - batch_processor.py:167 - INFO: Processed instances: 700\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:01,757 - batch_processor.py:167 - INFO: Processed instances: 800\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:02,025 - batch_processor.py:167 - INFO: Processed instances: 900\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:02,303 - batch_processor.py:167 - INFO: Processed instances: 1000\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:02,563 - batch_processor.py:167 - INFO: Processed instances: 1100\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:02,864 - batch_processor.py:167 - INFO: Processed instances: 1200\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:03,151 - batch_processor.py:167 - INFO: Processed instances: 1300\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:03,447 - batch_processor.py:167 - INFO: Processed instances: 1400\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:03,848 - batch_processor.py:167 - INFO: Processed instances: 1500\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:04,244 - batch_processor.py:167 - INFO: Processed instances: 1600\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:04,547 - batch_processor.py:167 - INFO: Processed instances: 1700\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:04,783 - batch_processor.py:167 - INFO: Processed instances: 1800\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:05,080 - batch_processor.py:167 - INFO: Processed instances: 1900\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:05,478 - batch_processor.py:167 - INFO: Processed instances: 2000\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:06,074 - batch_processor.py:167 - INFO: Processed instances: 2100\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:06,438 - batch_processor.py:167 - INFO: Processed instances: 2200\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:06,817 - batch_processor.py:167 - INFO: Processed instances: 2300\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:07,101 - batch_processor.py:167 - INFO: Processed instances: 2400\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:07,312 - batch_processor.py:167 - INFO: Processed instances: 2500\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:07,538 - batch_processor.py:167 - INFO: Processed instances: 2600\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:07,766 - batch_processor.py:167 - INFO: Processed instances: 2700\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:07,979 - batch_processor.py:167 - INFO: Processed instances: 2800\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:08,265 - batch_processor.py:167 - INFO: Processed instances: 2900\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:08,565 - batch_processor.py:167 - INFO: Processed instances: 3000\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:08,820 - batch_processor.py:167 - INFO: Processed instances: 3100\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:09,069 - batch_processor.py:167 - INFO: Processed instances: 3200\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:09,326 - batch_processor.py:167 - INFO: Processed instances: 3300\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:09,566 - batch_processor.py:167 - INFO: Processed instances: 3400\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:09,786 - batch_processor.py:167 - INFO: Processed instances: 3500\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:09,993 - batch_processor.py:167 - INFO: Processed instances: 3600\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:10,209 - batch_processor.py:167 - INFO: Processed instances: 3700\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:10,487 - batch_processor.py:167 - INFO: Processed instances: 3800\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:10,757 - batch_processor.py:167 - INFO: Processed instances: 3900\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:10,967 - batch_processor.py:167 - INFO: Processed instances: 4000\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:11,221 - batch_processor.py:167 - INFO: Processed instances: 4100\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:11,449 - batch_processor.py:167 - INFO: Processed instances: 4200\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:11,705 - batch_processor.py:167 - INFO: Processed instances: 4300\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:11,914 - batch_processor.py:167 - INFO: Processed instances: 4400\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:12,130 - batch_processor.py:167 - INFO: Processed instances: 4500\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:12,345 - batch_processor.py:167 - INFO: Processed instances: 4600\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:12,621 - batch_processor.py:167 - INFO: Processed instances: 4700\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:12,962 - batch_processor.py:167 - INFO: Processed instances: 4800\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:13,219 - batch_processor.py:167 - INFO: Processed instances: 4900\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:13,463 - batch_processor.py:167 - INFO: Processed instances: 5000\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:13,730 - batch_processor.py:167 - INFO: Processed instances: 5100\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:13,966 - batch_processor.py:167 - INFO: Processed instances: 5200\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:14,246 - batch_processor.py:167 - INFO: Processed instances: 5300\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:14,479 - batch_processor.py:167 - INFO: Processed instances: 5400\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:14,764 - batch_processor.py:167 - INFO: Processed instances: 5500\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:15,041 - batch_processor.py:167 - INFO: Processed instances: 5600\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:15,319 - batch_processor.py:167 - INFO: Processed instances: 5700\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:15,561 - batch_processor.py:167 - INFO: Processed instances: 5800\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:15,795 - batch_processor.py:167 - INFO: Processed instances: 5900\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:16,038 - batch_processor.py:167 - INFO: Processed instances: 6000\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:16,284 - batch_processor.py:167 - INFO: Processed instances: 6100\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:16,565 - batch_processor.py:167 - INFO: Processed instances: 6200\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:16,862 - batch_processor.py:167 - INFO: Processed instances: 6300\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:17,145 - batch_processor.py:167 - INFO: Processed instances: 6400\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:17,416 - batch_processor.py:167 - INFO: Processed instances: 6500\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:17,669 - batch_processor.py:167 - INFO: Processed instances: 6600\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:17,881 - batch_processor.py:167 - INFO: Processed instances: 6700\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:18,096 - batch_processor.py:167 - INFO: Processed instances: 6800\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:18,357 - batch_processor.py:167 - INFO: Processed instances: 6900\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:18,609 - batch_processor.py:167 - INFO: Processed instances: 7000\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:18,996 - batch_processor.py:167 - INFO: Processed instances: 7100\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:19,426 - batch_processor.py:167 - INFO: Processed instances: 7200\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:19,804 - batch_processor.py:167 - INFO: Processed instances: 7300\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:20,139 - batch_processor.py:167 - INFO: Processed instances: 7400\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:20,528 - batch_processor.py:167 - INFO: Processed instances: 7500\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:21,047 - batch_processor.py:167 - INFO: Processed instances: 7600\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:21,439 - batch_processor.py:167 - INFO: Processed instances: 7700\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:22,050 - batch_processor.py:167 - INFO: Processed instances: 7800\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:22,435 - batch_processor.py:167 - INFO: Processed instances: 7900\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:22,812 - batch_processor.py:167 - INFO: Processed instances: 8000\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:23,226 - batch_processor.py:167 - INFO: Processed instances: 8100\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:23,547 - batch_processor.py:167 - INFO: Processed instances: 8200\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:23,898 - batch_processor.py:167 - INFO: Processed instances: 8300\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:24,308 - batch_processor.py:167 - INFO: Processed instances: 8400\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:24,672 - batch_processor.py:167 - INFO: Processed instances: 8500\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:24,986 - batch_processor.py:167 - INFO: Processed instances: 8600\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:25,284 - batch_processor.py:167 - INFO: Processed instances: 8700\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:25,545 - batch_processor.py:167 - INFO: Processed instances: 8800\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:25,851 - batch_processor.py:167 - INFO: Processed instances: 8900\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:26,124 - batch_processor.py:167 - INFO: Processed instances: 9000\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:26,445 - batch_processor.py:167 - INFO: Processed instances: 9100\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:26,817 - batch_processor.py:167 - INFO: Processed instances: 9200\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:27,165 - batch_processor.py:167 - INFO: Processed instances: 9300\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:27,483 - batch_processor.py:167 - INFO: Processed instances: 9400\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:27,794 - batch_processor.py:167 - INFO: Processed instances: 9500\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:28,099 - batch_processor.py:167 - INFO: Processed instances: 9600\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:28,479 - batch_processor.py:167 - INFO: Processed instances: 9700\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:28,912 - batch_processor.py:167 - INFO: Processed instances: 9800\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:29,465 - batch_processor.py:167 - INFO: Processed instances: 9900\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:30,012 - batch_processor.py:167 - INFO: Processed instances: 10000\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:30,012 - batch_processor.py:168 - INFO: Total processed instances: 10000\u001b[0m\n", + "\u001b[33mseldon-batch-process-2033515954: 2021-01-15 11:46:30,012 - batch_processor.py:116 - INFO: Elapsed time: 30.899641513824463\u001b[0m\n", + "\u001b[35mseldon-batch-process-2070809024: seldondeployment.machinelearning.seldon.io \"sklearn\" deleted\u001b[0m\n" ] } ], "source": [ - "!argo logs -w seldon-batch-process || argo logs seldon-batch-process # The 2nd command is for argo 2.8+" + "!argo -n default logs seldon-batch-process" ] }, { @@ -419,41 +551,41 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Workflow ID is 323b666e-c546-431f-a64d-5fc780d68a18\n" + "Workflow UID is 511f64a2-0699-42eb-897a-c0a57b24072c\n" ] } ], "source": [ "import json\n", - "wf_arr = !argo get seldon-batch-process -o json\n", + "wf_arr = !argo get -n default seldon-batch-process -o json\n", "wf = json.loads(\"\".join(wf_arr))\n", - "WF_ID = wf[\"metadata\"][\"uid\"]\n", - "print(f\"Workflow ID is {WF_ID}\")" + "WF_UID = wf[\"metadata\"][\"uid\"]\n", + "print(f\"Workflow UID is {WF_UID}\")" ] }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[m\u001b[32m[2020-11-19 15:11:17 GMT]\u001b[0m\u001b[33m 2.7MiB\u001b[0m\u001b[1m output-data-323b666e-c546-431f-a64d-5fc780d68a18.txt\u001b[0m\r\n", + "\u001b[m\u001b[32m[2021-01-15 11:46:42 GMT] \u001b[0m\u001b[33m 3.4MiB \u001b[0moutput-data-511f64a2-0699-42eb-897a-c0a57b24072c.txt\n", "\u001b[0m" ] } ], "source": [ - "!mc ls minio-seldon/data/output-data-\"$WF_ID\".txt" + "!mc ls minio-seldon/data/output-data-\"$WF_UID\".txt" ] }, { @@ -465,54 +597,47 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "...68a18.txt: 2.75 MiB / 2.75 MiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 159.35 MiB/s 0s\u001b[0m\u001b[0m{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, \"meta\": {\"tags\": {\"tags\": {\"batch_id\": \"60b44a9c-2a79-11eb-b0aa-820f6474333b\", \"batch_index\": 0.0, \"batch_instance_id\": \"60b49614-2a79-11eb-9b0d-820f6474333b\"}}}}\n", - "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, \"meta\": {\"tags\": {\"tags\": {\"batch_id\": \"60b44a9c-2a79-11eb-b0aa-820f6474333b\", \"batch_index\": 3.0, \"batch_instance_id\": \"60b50932-2a79-11eb-9b0d-820f6474333b\"}}}}\n", - "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, \"meta\": {\"tags\": {\"tags\": {\"batch_id\": \"60b44a9c-2a79-11eb-b0aa-820f6474333b\", \"batch_index\": 1.0, \"batch_instance_id\": \"60b49d9e-2a79-11eb-9b0d-820f6474333b\"}}}}\n", - "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, \"meta\": {\"tags\": {\"tags\": {\"batch_id\": \"60b44a9c-2a79-11eb-b0aa-820f6474333b\", \"batch_index\": 2.0, \"batch_instance_id\": \"60b4e60a-2a79-11eb-9b0d-820f6474333b\"}}}}\n", - "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, \"meta\": {\"tags\": {\"tags\": {\"batch_id\": \"60b44a9c-2a79-11eb-b0aa-820f6474333b\", \"batch_index\": 5.0, \"batch_instance_id\": \"60b5269c-2a79-11eb-9b0d-820f6474333b\"}}}}\n", - "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, \"meta\": {\"tags\": {\"tags\": {\"batch_id\": \"60b44a9c-2a79-11eb-b0aa-820f6474333b\", \"batch_index\": 9.0, \"batch_instance_id\": \"60b542a8-2a79-11eb-9b0d-820f6474333b\"}}}}\n", - "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, \"meta\": {\"tags\": {\"tags\": {\"batch_id\": \"60b44a9c-2a79-11eb-b0aa-820f6474333b\", \"batch_index\": 7.0, \"batch_instance_id\": \"60b53fce-2a79-11eb-9b0d-820f6474333b\"}}}}\n", - "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, \"meta\": {\"tags\": {\"tags\": {\"batch_id\": \"60b44a9c-2a79-11eb-b0aa-820f6474333b\", \"batch_index\": 6.0, \"batch_instance_id\": \"60b53e7a-2a79-11eb-9b0d-820f6474333b\"}}}}\n", - "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, \"meta\": {\"tags\": {\"tags\": {\"batch_id\": \"60b44a9c-2a79-11eb-b0aa-820f6474333b\", \"batch_index\": 8.0, \"batch_instance_id\": \"60b540d2-2a79-11eb-9b0d-820f6474333b\"}}}}\n", - "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, \"meta\": {\"tags\": {\"tags\": {\"batch_id\": \"60b44a9c-2a79-11eb-b0aa-820f6474333b\", \"batch_index\": 11.0, \"batch_instance_id\": \"60b546c2-2a79-11eb-9b0d-820f6474333b\"}}}}\n" + "...4072c.txt: 3.36 MiB / 3.36 MiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 192.59 MiB/s 0s\u001b[0m\u001b[0m\u001b[m\u001b[32;1m{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.1859090109477526, 0.46433848375587844, 0.349752505296369]]}, \"meta\": {\"requestPath\": {\"classifier\": \"seldonio/sklearnserver:1.6.0-dev\"}, \"tags\": {\"tags\": {\"batch_id\": \"3c4000b8-5727-11eb-91c1-6e88dc41eb63\", \"batch_index\": 1.0, \"batch_instance_id\": \"3c40e1e0-5727-11eb-9fe5-6e88dc41eb63\"}}}}\n", + "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.1679456497678022, 0.42318259169768935, 0.4088717585345084]]}, \"meta\": {\"requestPath\": {\"classifier\": \"seldonio/sklearnserver:1.6.0-dev\"}, \"tags\": {\"tags\": {\"batch_id\": \"3c4000b8-5727-11eb-91c1-6e88dc41eb63\", \"batch_index\": 22.0, \"batch_instance_id\": \"3c42efb2-5727-11eb-9fe5-6e88dc41eb63\"}}}}\n", + "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.5329356306409886, 0.2531124742231082, 0.21395189513590318]]}, \"meta\": {\"requestPath\": {\"classifier\": \"seldonio/sklearnserver:1.6.0-dev\"}, \"tags\": {\"tags\": {\"batch_id\": \"3c4000b8-5727-11eb-91c1-6e88dc41eb63\", \"batch_index\": 25.0, \"batch_instance_id\": \"3c43dac6-5727-11eb-9fe5-6e88dc41eb63\"}}}}\n", + "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.5057216294927378, 0.37562353221834527, 0.11865483828891676]]}, \"meta\": {\"requestPath\": {\"classifier\": \"seldonio/sklearnserver:1.6.0-dev\"}, \"tags\": {\"tags\": {\"batch_id\": \"3c4000b8-5727-11eb-91c1-6e88dc41eb63\", \"batch_index\": 20.0, \"batch_instance_id\": \"3c4294a4-5727-11eb-9fe5-6e88dc41eb63\"}}}}\n", + "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.16020781530738484, 0.49084414063547427, 0.3489480440571409]]}, \"meta\": {\"requestPath\": {\"classifier\": \"seldonio/sklearnserver:1.6.0-dev\"}, \"tags\": {\"tags\": {\"batch_id\": \"3c4000b8-5727-11eb-91c1-6e88dc41eb63\", \"batch_index\": 24.0, \"batch_instance_id\": \"3c439a34-5727-11eb-9fe5-6e88dc41eb63\"}}}}\n", + "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.49551509682202705, 0.4192462053867995, 0.08523869779117352]]}, \"meta\": {\"requestPath\": {\"classifier\": \"seldonio/sklearnserver:1.6.0-dev\"}, \"tags\": {\"tags\": {\"batch_id\": \"3c4000b8-5727-11eb-91c1-6e88dc41eb63\", \"batch_index\": 0.0, \"batch_instance_id\": \"3c40c6d8-5727-11eb-9fe5-6e88dc41eb63\"}}}}\n", + "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.17817271417040353, 0.4160568279837039, 0.4057704578458926]]}, \"meta\": {\"requestPath\": {\"classifier\": \"seldonio/sklearnserver:1.6.0-dev\"}, \"tags\": {\"tags\": {\"batch_id\": \"3c4000b8-5727-11eb-91c1-6e88dc41eb63\", \"batch_index\": 6.0, \"batch_instance_id\": \"3c41aa58-5727-11eb-9fe5-6e88dc41eb63\"}}}}\n", + "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.31086648314817084, 0.43371070280306884, 0.25542281404876027]]}, \"meta\": {\"requestPath\": {\"classifier\": \"seldonio/sklearnserver:1.6.0-dev\"}, \"tags\": {\"tags\": {\"batch_id\": \"3c4000b8-5727-11eb-91c1-6e88dc41eb63\", \"batch_index\": 27.0, \"batch_instance_id\": \"3c44420e-5727-11eb-9fe5-6e88dc41eb63\"}}}}\n", + "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.4381942165350952, 0.39483980719426687, 0.16696597627063794]]}, \"meta\": {\"requestPath\": {\"classifier\": \"seldonio/sklearnserver:1.6.0-dev\"}, \"tags\": {\"tags\": {\"batch_id\": \"3c4000b8-5727-11eb-91c1-6e88dc41eb63\", \"batch_index\": 34.0, \"batch_instance_id\": \"3c448ff2-5727-11eb-9fe5-6e88dc41eb63\"}}}}\n", + "{\"data\": {\"names\": [\"t:0\", \"t:1\", \"t:2\"], \"ndarray\": [[0.2975075875929912, 0.25439317776178244, 0.44809923464522644]]}, \"meta\": {\"requestPath\": {\"classifier\": \"seldonio/sklearnserver:1.6.0-dev\"}, \"tags\": {\"tags\": {\"batch_id\": \"3c4000b8-5727-11eb-91c1-6e88dc41eb63\", \"batch_index\": 4.0, \"batch_instance_id\": \"3c41837a-5727-11eb-9fe5-6e88dc41eb63\"}}}}\n" ] } ], "source": [ - "!mc cp minio-seldon/data/output-data-\"$WF_ID\".txt assets/output-data.txt\n", + "!mc cp minio-seldon/data/output-data-\"$WF_UID\".txt assets/output-data.txt\n", "!head assets/output-data.txt" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Workflow 'seldon-batch-process' deleted\r\n" + "Workflow 'seldon-batch-process' deleted\n" ] } ], "source": [ - "!argo delete seldon-batch-process" + "!argo delete -n default seldon-batch-process" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -531,7 +656,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.3" + "version": "3.7.9" } }, "nbformat": 4, diff --git a/examples/batch/argo-workflows-batch/README.md b/examples/batch/argo-workflows-batch/README.md index f12328b7eb..292dd1708b 100644 --- a/examples/batch/argo-workflows-batch/README.md +++ b/examples/batch/argo-workflows-batch/README.md @@ -8,39 +8,58 @@ Dependencies: * Minio running in your cluster to use as local (s3) object storage * Argo Workfklows installed in cluster (and argo CLI for commands) - ### Setup #### Install Seldon Core Use the notebook to [set-up Seldon Core with Ambassador or Istio Ingress](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html). -ote: If running with KIND you need to make sure do follow [these steps](https://github.com/argoproj/argo/issues/2376#issuecomment-595593237) as workaround to the `/.../docker.sock` known issue. +Note: If running with KIND you need to make sure do follow [these steps](https://github.com/argoproj/argo/issues/2376#issuecomment-595593237) as workaround to the `/.../docker.sock` known issue. + +#### Set up Minio in your cluster +Use the notebook to [set-up Minio in your cluster](https://docs.seldon.io/projects/seldon-core/en/latest/examples/minio_setup.html). -### Set up Minio in your cluster -se the notebook to [set-up Minio in your cluster](https://docs.seldon.io/projects/seldon-core/en/latest/examples/minio_setup.html). +#### Create rclone configuration +In this example, our workflow stages responsible for pulling / pushing data to in-cluster MinIO S3 storage will use `rclone` CLI. +In order to configure the CLI we will create a following secret: -### Copy the Minio Secret to namespace -We need to re-use the minio secret for the batch job, so this can be done by just copying the minio secret created in the `minio-system` +```python +%%writefile rclone-config.yaml +apiVersion: v1 +kind: Secret +metadata: + name: rclone-config-secret +type: Opaque +stringData: + rclone.conf: | + [cluster-minio] + type = s3 + provider = minio + env_auth = false + access_key_id = minioadmin + secret_access_key = minioadmin + endpoint = http://minio.minio-system.svc.cluster.local:9000 +``` + + Overwriting rclone-config.yaml -The command below just copies the secred with the name "minio" from the minio-system namespace to the default namespace. ```python -!kubectl get secret minio -n minio-system -o json | jq '{apiVersion,data,kind,metadata,type} | .metadata |= {"annotations", "name"}' | kubectl apply -n default -f - +!kubectl apply -n default -f rclone-config.yaml ``` - secret/minio created + secret/rclone-config-secret created #### Install Argo Workflows You can follow the instructions from the official [Argo Workflows Documentation](https://github.com/argoproj/argo#quickstart). -You also need to make sure that argo has permissions to create seldon deployments - for this you can a role: +You also need to make sure that argo has permissions to create seldon deployments - for this you can create a role: ```python -!kubectl create -f - < `/assets/input-data.txt` - download-object-store: Total: 0 B, Transferred: 146.48 KiB, Speed: 31.81 MiB/s - process-batch-inputs: Elapsed time: 35.089903831481934 - upload-object-store: Added `minio-local` successfully. - upload-object-store: `/assets/output-data.txt` -> `minio-local/data/output-data-401c8bc0-0ff0-4f7b-94ba-347df5c786f9.txt` - upload-object-store: Total: 0 B, Transferred: 2.75 MiB, Speed: 105.34 MiB/s + seldon-batch-process-3626514072: time="2021-01-15T11:44:57.620Z" level=info msg="Starting Workflow Executor" version=v2.12.3 + seldon-batch-process-3626514072: time="2021-01-15T11:44:57.622Z" level=info msg="Creating a K8sAPI executor" + seldon-batch-process-3626514072: time="2021-01-15T11:44:57.622Z" level=info msg="Executor (version: v2.12.3, build_date: 2021-01-05T00:54:54Z) initialized (pod: default/seldon-batch-process-3626514072) with template:\n{\"name\":\"create-seldon-resource-template\",\"arguments\":{},\"inputs\":{},\"outputs\":{},\"metadata\":{\"annotations\":{\"sidecar.istio.io/inject\":\"false\"}},\"resource\":{\"action\":\"create\",\"manifest\":\"apiVersion: machinelearning.seldon.io/v1\\nkind: SeldonDeployment\\nmetadata:\\n name: \\\"sklearn\\\"\\n namespace: default\\n ownerReferences:\\n - apiVersion: argoproj.io/v1alpha1\\n blockOwnerDeletion: true\\n kind: Workflow\\n name: \\\"seldon-batch-process\\\"\\n uid: \\\"511f64a2-0699-42eb-897a-c0a57b24072c\\\"\\nspec:\\n name: \\\"sklearn\\\"\\n predictors:\\n - componentSpecs:\\n - spec:\\n containers:\\n - name: classifier\\n env:\\n - name: GUNICORN_THREADS\\n value: 10\\n - name: GUNICORN_WORKERS\\n value: 1\\n resources:\\n requests:\\n cpu: 50m\\n memory: 100Mi\\n limits:\\n cpu: 50m\\n memory: 1000Mi\\n graph:\\n children: []\\n implementation: SKLEARN_SERVER\\n modelUri: gs://seldon-models/sklearn/iris\\n name: classifier\\n name: default\\n replicas: 10\\n\"}}" + seldon-batch-process-3626514072: time="2021-01-15T11:44:57.622Z" level=info msg="Loading manifest to /tmp/manifest.yaml" + seldon-batch-process-3626514072: time="2021-01-15T11:44:57.622Z" level=info msg="kubectl create -f /tmp/manifest.yaml -o json" + seldon-batch-process-3626514072: time="2021-01-15T11:44:58.044Z" level=info msg=default/SeldonDeployment.machinelearning.seldon.io/sklearn + seldon-batch-process-3626514072: time="2021-01-15T11:44:58.044Z" level=info msg="Starting SIGUSR2 signal monitor" + seldon-batch-process-3626514072: time="2021-01-15T11:44:58.045Z" level=info msg="No output parameters" + seldon-batch-process-2052519094: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 0 of 10 updated replicas are available... + seldon-batch-process-2052519094: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 1 of 10 updated replicas are available... + seldon-batch-process-2052519094: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 2 of 10 updated replicas are available... + seldon-batch-process-2052519094: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 3 of 10 updated replicas are available... + seldon-batch-process-2052519094: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 4 of 10 updated replicas are available... + seldon-batch-process-2052519094: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 5 of 10 updated replicas are available... + seldon-batch-process-2052519094: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 6 of 10 updated replicas are available... + seldon-batch-process-2052519094: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 7 of 10 updated replicas are available... + seldon-batch-process-2052519094: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 8 of 10 updated replicas are available... + seldon-batch-process-2052519094: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 9 of 10 updated replicas are available... + seldon-batch-process-2052519094: deployment "sklearn-default-0-classifier" successfully rolled out + seldon-batch-process-2033515954: 2021-01-15 11:46:00,306 - batch_processor.py:167 - INFO: Processed instances: 100 + seldon-batch-process-2033515954: 2021-01-15 11:46:00,321 - batch_processor.py:167 - INFO: Processed instances: 200 + seldon-batch-process-2033515954: 2021-01-15 11:46:00,411 - batch_processor.py:167 - INFO: Processed instances: 300 + seldon-batch-process-2033515954: 2021-01-15 11:46:00,464 - batch_processor.py:167 - INFO: Processed instances: 400 + seldon-batch-process-2033515954: 2021-01-15 11:46:00,768 - batch_processor.py:167 - INFO: Processed instances: 500 + seldon-batch-process-2033515954: 2021-01-15 11:46:01,055 - batch_processor.py:167 - INFO: Processed instances: 600 + seldon-batch-process-2033515954: 2021-01-15 11:46:01,439 - batch_processor.py:167 - INFO: Processed instances: 700 + seldon-batch-process-2033515954: 2021-01-15 11:46:01,757 - batch_processor.py:167 - INFO: Processed instances: 800 + seldon-batch-process-2033515954: 2021-01-15 11:46:02,025 - batch_processor.py:167 - INFO: Processed instances: 900 + seldon-batch-process-2033515954: 2021-01-15 11:46:02,303 - batch_processor.py:167 - INFO: Processed instances: 1000 + seldon-batch-process-2033515954: 2021-01-15 11:46:02,563 - batch_processor.py:167 - INFO: Processed instances: 1100 + seldon-batch-process-2033515954: 2021-01-15 11:46:02,864 - batch_processor.py:167 - INFO: Processed instances: 1200 + seldon-batch-process-2033515954: 2021-01-15 11:46:03,151 - batch_processor.py:167 - INFO: Processed instances: 1300 + seldon-batch-process-2033515954: 2021-01-15 11:46:03,447 - batch_processor.py:167 - INFO: Processed instances: 1400 + seldon-batch-process-2033515954: 2021-01-15 11:46:03,848 - batch_processor.py:167 - INFO: Processed instances: 1500 + seldon-batch-process-2033515954: 2021-01-15 11:46:04,244 - batch_processor.py:167 - INFO: Processed instances: 1600 + seldon-batch-process-2033515954: 2021-01-15 11:46:04,547 - batch_processor.py:167 - INFO: Processed instances: 1700 + seldon-batch-process-2033515954: 2021-01-15 11:46:04,783 - batch_processor.py:167 - INFO: Processed instances: 1800 + seldon-batch-process-2033515954: 2021-01-15 11:46:05,080 - batch_processor.py:167 - INFO: Processed instances: 1900 + seldon-batch-process-2033515954: 2021-01-15 11:46:05,478 - batch_processor.py:167 - INFO: Processed instances: 2000 + seldon-batch-process-2033515954: 2021-01-15 11:46:06,074 - batch_processor.py:167 - INFO: Processed instances: 2100 + seldon-batch-process-2033515954: 2021-01-15 11:46:06,438 - batch_processor.py:167 - INFO: Processed instances: 2200 + seldon-batch-process-2033515954: 2021-01-15 11:46:06,817 - batch_processor.py:167 - INFO: Processed instances: 2300 + seldon-batch-process-2033515954: 2021-01-15 11:46:07,101 - batch_processor.py:167 - INFO: Processed instances: 2400 + seldon-batch-process-2033515954: 2021-01-15 11:46:07,312 - batch_processor.py:167 - INFO: Processed instances: 2500 + seldon-batch-process-2033515954: 2021-01-15 11:46:07,538 - batch_processor.py:167 - INFO: Processed instances: 2600 + seldon-batch-process-2033515954: 2021-01-15 11:46:07,766 - batch_processor.py:167 - INFO: Processed instances: 2700 + seldon-batch-process-2033515954: 2021-01-15 11:46:07,979 - batch_processor.py:167 - INFO: Processed instances: 2800 + seldon-batch-process-2033515954: 2021-01-15 11:46:08,265 - batch_processor.py:167 - INFO: Processed instances: 2900 + seldon-batch-process-2033515954: 2021-01-15 11:46:08,565 - batch_processor.py:167 - INFO: Processed instances: 3000 + seldon-batch-process-2033515954: 2021-01-15 11:46:08,820 - batch_processor.py:167 - INFO: Processed instances: 3100 + seldon-batch-process-2033515954: 2021-01-15 11:46:09,069 - batch_processor.py:167 - INFO: Processed instances: 3200 + seldon-batch-process-2033515954: 2021-01-15 11:46:09,326 - batch_processor.py:167 - INFO: Processed instances: 3300 + seldon-batch-process-2033515954: 2021-01-15 11:46:09,566 - batch_processor.py:167 - INFO: Processed instances: 3400 + seldon-batch-process-2033515954: 2021-01-15 11:46:09,786 - batch_processor.py:167 - INFO: Processed instances: 3500 + seldon-batch-process-2033515954: 2021-01-15 11:46:09,993 - batch_processor.py:167 - INFO: Processed instances: 3600 + seldon-batch-process-2033515954: 2021-01-15 11:46:10,209 - batch_processor.py:167 - INFO: Processed instances: 3700 + seldon-batch-process-2033515954: 2021-01-15 11:46:10,487 - batch_processor.py:167 - INFO: Processed instances: 3800 + seldon-batch-process-2033515954: 2021-01-15 11:46:10,757 - batch_processor.py:167 - INFO: Processed instances: 3900 + seldon-batch-process-2033515954: 2021-01-15 11:46:10,967 - batch_processor.py:167 - INFO: Processed instances: 4000 + seldon-batch-process-2033515954: 2021-01-15 11:46:11,221 - batch_processor.py:167 - INFO: Processed instances: 4100 + seldon-batch-process-2033515954: 2021-01-15 11:46:11,449 - batch_processor.py:167 - INFO: Processed instances: 4200 + seldon-batch-process-2033515954: 2021-01-15 11:46:11,705 - batch_processor.py:167 - INFO: Processed instances: 4300 + seldon-batch-process-2033515954: 2021-01-15 11:46:11,914 - batch_processor.py:167 - INFO: Processed instances: 4400 + seldon-batch-process-2033515954: 2021-01-15 11:46:12,130 - batch_processor.py:167 - INFO: Processed instances: 4500 + seldon-batch-process-2033515954: 2021-01-15 11:46:12,345 - batch_processor.py:167 - INFO: Processed instances: 4600 + seldon-batch-process-2033515954: 2021-01-15 11:46:12,621 - batch_processor.py:167 - INFO: Processed instances: 4700 + seldon-batch-process-2033515954: 2021-01-15 11:46:12,962 - batch_processor.py:167 - INFO: Processed instances: 4800 + seldon-batch-process-2033515954: 2021-01-15 11:46:13,219 - batch_processor.py:167 - INFO: Processed instances: 4900 + seldon-batch-process-2033515954: 2021-01-15 11:46:13,463 - batch_processor.py:167 - INFO: Processed instances: 5000 + seldon-batch-process-2033515954: 2021-01-15 11:46:13,730 - batch_processor.py:167 - INFO: Processed instances: 5100 + seldon-batch-process-2033515954: 2021-01-15 11:46:13,966 - batch_processor.py:167 - INFO: Processed instances: 5200 + seldon-batch-process-2033515954: 2021-01-15 11:46:14,246 - batch_processor.py:167 - INFO: Processed instances: 5300 + seldon-batch-process-2033515954: 2021-01-15 11:46:14,479 - batch_processor.py:167 - INFO: Processed instances: 5400 + seldon-batch-process-2033515954: 2021-01-15 11:46:14,764 - batch_processor.py:167 - INFO: Processed instances: 5500 + seldon-batch-process-2033515954: 2021-01-15 11:46:15,041 - batch_processor.py:167 - INFO: Processed instances: 5600 + seldon-batch-process-2033515954: 2021-01-15 11:46:15,319 - batch_processor.py:167 - INFO: Processed instances: 5700 + seldon-batch-process-2033515954: 2021-01-15 11:46:15,561 - batch_processor.py:167 - INFO: Processed instances: 5800 + seldon-batch-process-2033515954: 2021-01-15 11:46:15,795 - batch_processor.py:167 - INFO: Processed instances: 5900 + seldon-batch-process-2033515954: 2021-01-15 11:46:16,038 - batch_processor.py:167 - INFO: Processed instances: 6000 + seldon-batch-process-2033515954: 2021-01-15 11:46:16,284 - batch_processor.py:167 - INFO: Processed instances: 6100 + seldon-batch-process-2033515954: 2021-01-15 11:46:16,565 - batch_processor.py:167 - INFO: Processed instances: 6200 + seldon-batch-process-2033515954: 2021-01-15 11:46:16,862 - batch_processor.py:167 - INFO: Processed instances: 6300 + seldon-batch-process-2033515954: 2021-01-15 11:46:17,145 - batch_processor.py:167 - INFO: Processed instances: 6400 + seldon-batch-process-2033515954: 2021-01-15 11:46:17,416 - batch_processor.py:167 - INFO: Processed instances: 6500 + seldon-batch-process-2033515954: 2021-01-15 11:46:17,669 - batch_processor.py:167 - INFO: Processed instances: 6600 + seldon-batch-process-2033515954: 2021-01-15 11:46:17,881 - batch_processor.py:167 - INFO: Processed instances: 6700 + seldon-batch-process-2033515954: 2021-01-15 11:46:18,096 - batch_processor.py:167 - INFO: Processed instances: 6800 + seldon-batch-process-2033515954: 2021-01-15 11:46:18,357 - batch_processor.py:167 - INFO: Processed instances: 6900 + seldon-batch-process-2033515954: 2021-01-15 11:46:18,609 - batch_processor.py:167 - INFO: Processed instances: 7000 + seldon-batch-process-2033515954: 2021-01-15 11:46:18,996 - batch_processor.py:167 - INFO: Processed instances: 7100 + seldon-batch-process-2033515954: 2021-01-15 11:46:19,426 - batch_processor.py:167 - INFO: Processed instances: 7200 + seldon-batch-process-2033515954: 2021-01-15 11:46:19,804 - batch_processor.py:167 - INFO: Processed instances: 7300 + seldon-batch-process-2033515954: 2021-01-15 11:46:20,139 - batch_processor.py:167 - INFO: Processed instances: 7400 + seldon-batch-process-2033515954: 2021-01-15 11:46:20,528 - batch_processor.py:167 - INFO: Processed instances: 7500 + seldon-batch-process-2033515954: 2021-01-15 11:46:21,047 - batch_processor.py:167 - INFO: Processed instances: 7600 + seldon-batch-process-2033515954: 2021-01-15 11:46:21,439 - batch_processor.py:167 - INFO: Processed instances: 7700 + seldon-batch-process-2033515954: 2021-01-15 11:46:22,050 - batch_processor.py:167 - INFO: Processed instances: 7800 + seldon-batch-process-2033515954: 2021-01-15 11:46:22,435 - batch_processor.py:167 - INFO: Processed instances: 7900 + seldon-batch-process-2033515954: 2021-01-15 11:46:22,812 - batch_processor.py:167 - INFO: Processed instances: 8000 + seldon-batch-process-2033515954: 2021-01-15 11:46:23,226 - batch_processor.py:167 - INFO: Processed instances: 8100 + seldon-batch-process-2033515954: 2021-01-15 11:46:23,547 - batch_processor.py:167 - INFO: Processed instances: 8200 + seldon-batch-process-2033515954: 2021-01-15 11:46:23,898 - batch_processor.py:167 - INFO: Processed instances: 8300 + seldon-batch-process-2033515954: 2021-01-15 11:46:24,308 - batch_processor.py:167 - INFO: Processed instances: 8400 + seldon-batch-process-2033515954: 2021-01-15 11:46:24,672 - batch_processor.py:167 - INFO: Processed instances: 8500 + seldon-batch-process-2033515954: 2021-01-15 11:46:24,986 - batch_processor.py:167 - INFO: Processed instances: 8600 + seldon-batch-process-2033515954: 2021-01-15 11:46:25,284 - batch_processor.py:167 - INFO: Processed instances: 8700 + seldon-batch-process-2033515954: 2021-01-15 11:46:25,545 - batch_processor.py:167 - INFO: Processed instances: 8800 + seldon-batch-process-2033515954: 2021-01-15 11:46:25,851 - batch_processor.py:167 - INFO: Processed instances: 8900 + seldon-batch-process-2033515954: 2021-01-15 11:46:26,124 - batch_processor.py:167 - INFO: Processed instances: 9000 + seldon-batch-process-2033515954: 2021-01-15 11:46:26,445 - batch_processor.py:167 - INFO: Processed instances: 9100 + seldon-batch-process-2033515954: 2021-01-15 11:46:26,817 - batch_processor.py:167 - INFO: Processed instances: 9200 + seldon-batch-process-2033515954: 2021-01-15 11:46:27,165 - batch_processor.py:167 - INFO: Processed instances: 9300 + seldon-batch-process-2033515954: 2021-01-15 11:46:27,483 - batch_processor.py:167 - INFO: Processed instances: 9400 + seldon-batch-process-2033515954: 2021-01-15 11:46:27,794 - batch_processor.py:167 - INFO: Processed instances: 9500 + seldon-batch-process-2033515954: 2021-01-15 11:46:28,099 - batch_processor.py:167 - INFO: Processed instances: 9600 + seldon-batch-process-2033515954: 2021-01-15 11:46:28,479 - batch_processor.py:167 - INFO: Processed instances: 9700 + seldon-batch-process-2033515954: 2021-01-15 11:46:28,912 - batch_processor.py:167 - INFO: Processed instances: 9800 + seldon-batch-process-2033515954: 2021-01-15 11:46:29,465 - batch_processor.py:167 - INFO: Processed instances: 9900 + seldon-batch-process-2033515954: 2021-01-15 11:46:30,012 - batch_processor.py:167 - INFO: Processed instances: 10000 + seldon-batch-process-2033515954: 2021-01-15 11:46:30,012 - batch_processor.py:168 - INFO: Total processed instances: 10000 + seldon-batch-process-2033515954: 2021-01-15 11:46:30,012 - batch_processor.py:116 - INFO: Elapsed time: 30.899641513824463 + seldon-batch-process-2070809024: seldondeployment.machinelearning.seldon.io "sklearn" deleted ### Check output in object store @@ -233,52 +384,47 @@ First we can check that the file is present: ```python import json -wf_arr = !argo get seldon-batch-process -o json +wf_arr = !argo get -n default seldon-batch-process -o json wf = json.loads("".join(wf_arr)) -WF_ID = wf["metadata"]["uid"] -print(f"Workflow ID is {WF_ID}") +WF_UID = wf["metadata"]["uid"] +print(f"Workflow UID is {WF_UID}") ``` - Workflow ID is 401c8bc0-0ff0-4f7b-94ba-347df5c786f9 + Workflow UID is 511f64a2-0699-42eb-897a-c0a57b24072c ```python -!mc ls minio-seldon/data/output-data-"$WF_ID".txt +!mc ls minio-seldon/data/output-data-"$WF_UID".txt ``` - [2020-08-06 08:23:07 BST]  2.7MiB output-data-401c8bc0-0ff0-4f7b-94ba-347df5c786f9.txt + [2021-01-15 11:46:42 GMT]  3.4MiB output-data-511f64a2-0699-42eb-897a-c0a57b24072c.txt  Now we can output the contents of the file created using the `mc head` command. ```python -!mc cp minio-seldon/data/output-data-"$WF_ID".txt assets/output-data.txt +!mc cp minio-seldon/data/output-data-"$WF_UID".txt assets/output-data.txt !head assets/output-data.txt ``` - ...786f9.txt: 2.75 MiB / 2.75 MiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 26.55 MiB/s 0s{"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, "meta": {"tags": {"tags": {"batch_id": "95e6e8d0-d7b5-11ea-b00e-ea443eed4c19", "batch_index": 2.0, "batch_instance_id": "95e7df56-d7b5-11ea-b5f2-ea443eed4c19"}}}} - {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, "meta": {"tags": {"tags": {"batch_id": "95e6e8d0-d7b5-11ea-b00e-ea443eed4c19", "batch_index": 0.0, "batch_instance_id": "95e77c3c-d7b5-11ea-b5f2-ea443eed4c19"}}}} - {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, "meta": {"tags": {"tags": {"batch_id": "95e6e8d0-d7b5-11ea-b00e-ea443eed4c19", "batch_index": 1.0, "batch_instance_id": "95e787ae-d7b5-11ea-b5f2-ea443eed4c19"}}}} - {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, "meta": {"tags": {"tags": {"batch_id": "95e6e8d0-d7b5-11ea-b00e-ea443eed4c19", "batch_index": 3.0, "batch_instance_id": "95e80990-d7b5-11ea-b5f2-ea443eed4c19"}}}} - {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, "meta": {"tags": {"tags": {"batch_id": "95e6e8d0-d7b5-11ea-b00e-ea443eed4c19", "batch_index": 4.0, "batch_instance_id": "95e83cf8-d7b5-11ea-b5f2-ea443eed4c19"}}}} - {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, "meta": {"tags": {"tags": {"batch_id": "95e6e8d0-d7b5-11ea-b00e-ea443eed4c19", "batch_index": 6.0, "batch_instance_id": "95e85990-d7b5-11ea-b5f2-ea443eed4c19"}}}} - {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, "meta": {"tags": {"tags": {"batch_id": "95e6e8d0-d7b5-11ea-b00e-ea443eed4c19", "batch_index": 8.0, "batch_instance_id": "95e85e40-d7b5-11ea-b5f2-ea443eed4c19"}}}} - {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, "meta": {"tags": {"tags": {"batch_id": "95e6e8d0-d7b5-11ea-b00e-ea443eed4c19", "batch_index": 7.0, "batch_instance_id": "95e85c1a-d7b5-11ea-b5f2-ea443eed4c19"}}}} - {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, "meta": {"tags": {"tags": {"batch_id": "95e6e8d0-d7b5-11ea-b00e-ea443eed4c19", "batch_index": 10.0, "batch_instance_id": "95e864c6-d7b5-11ea-b5f2-ea443eed4c19"}}}} - {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.0006985194531162841, 0.003668039039435755, 0.9956334415074478]]}, "meta": {"tags": {"tags": {"batch_id": "95e6e8d0-d7b5-11ea-b00e-ea443eed4c19", "batch_index": 5.0, "batch_instance_id": "95e83f8c-d7b5-11ea-b5f2-ea443eed4c19"}}}} + ...4072c.txt: 3.36 MiB / 3.36 MiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 192.59 MiB/s 0s{"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.1859090109477526, 0.46433848375587844, 0.349752505296369]]}, "meta": {"requestPath": {"classifier": "seldonio/sklearnserver:1.6.0-dev"}, "tags": {"tags": {"batch_id": "3c4000b8-5727-11eb-91c1-6e88dc41eb63", "batch_index": 1.0, "batch_instance_id": "3c40e1e0-5727-11eb-9fe5-6e88dc41eb63"}}}} + {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.1679456497678022, 0.42318259169768935, 0.4088717585345084]]}, "meta": {"requestPath": {"classifier": "seldonio/sklearnserver:1.6.0-dev"}, "tags": {"tags": {"batch_id": "3c4000b8-5727-11eb-91c1-6e88dc41eb63", "batch_index": 22.0, "batch_instance_id": "3c42efb2-5727-11eb-9fe5-6e88dc41eb63"}}}} + {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.5329356306409886, 0.2531124742231082, 0.21395189513590318]]}, "meta": {"requestPath": {"classifier": "seldonio/sklearnserver:1.6.0-dev"}, "tags": {"tags": {"batch_id": "3c4000b8-5727-11eb-91c1-6e88dc41eb63", "batch_index": 25.0, "batch_instance_id": "3c43dac6-5727-11eb-9fe5-6e88dc41eb63"}}}} + {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.5057216294927378, 0.37562353221834527, 0.11865483828891676]]}, "meta": {"requestPath": {"classifier": "seldonio/sklearnserver:1.6.0-dev"}, "tags": {"tags": {"batch_id": "3c4000b8-5727-11eb-91c1-6e88dc41eb63", "batch_index": 20.0, "batch_instance_id": "3c4294a4-5727-11eb-9fe5-6e88dc41eb63"}}}} + {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.16020781530738484, 0.49084414063547427, 0.3489480440571409]]}, "meta": {"requestPath": {"classifier": "seldonio/sklearnserver:1.6.0-dev"}, "tags": {"tags": {"batch_id": "3c4000b8-5727-11eb-91c1-6e88dc41eb63", "batch_index": 24.0, "batch_instance_id": "3c439a34-5727-11eb-9fe5-6e88dc41eb63"}}}} + {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.49551509682202705, 0.4192462053867995, 0.08523869779117352]]}, "meta": {"requestPath": {"classifier": "seldonio/sklearnserver:1.6.0-dev"}, "tags": {"tags": {"batch_id": "3c4000b8-5727-11eb-91c1-6e88dc41eb63", "batch_index": 0.0, "batch_instance_id": "3c40c6d8-5727-11eb-9fe5-6e88dc41eb63"}}}} + {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.17817271417040353, 0.4160568279837039, 0.4057704578458926]]}, "meta": {"requestPath": {"classifier": "seldonio/sklearnserver:1.6.0-dev"}, "tags": {"tags": {"batch_id": "3c4000b8-5727-11eb-91c1-6e88dc41eb63", "batch_index": 6.0, "batch_instance_id": "3c41aa58-5727-11eb-9fe5-6e88dc41eb63"}}}} + {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.31086648314817084, 0.43371070280306884, 0.25542281404876027]]}, "meta": {"requestPath": {"classifier": "seldonio/sklearnserver:1.6.0-dev"}, "tags": {"tags": {"batch_id": "3c4000b8-5727-11eb-91c1-6e88dc41eb63", "batch_index": 27.0, "batch_instance_id": "3c44420e-5727-11eb-9fe5-6e88dc41eb63"}}}} + {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.4381942165350952, 0.39483980719426687, 0.16696597627063794]]}, "meta": {"requestPath": {"classifier": "seldonio/sklearnserver:1.6.0-dev"}, "tags": {"tags": {"batch_id": "3c4000b8-5727-11eb-91c1-6e88dc41eb63", "batch_index": 34.0, "batch_instance_id": "3c448ff2-5727-11eb-9fe5-6e88dc41eb63"}}}} + {"data": {"names": ["t:0", "t:1", "t:2"], "ndarray": [[0.2975075875929912, 0.25439317776178244, 0.44809923464522644]]}, "meta": {"requestPath": {"classifier": "seldonio/sklearnserver:1.6.0-dev"}, "tags": {"tags": {"batch_id": "3c4000b8-5727-11eb-91c1-6e88dc41eb63", "batch_index": 4.0, "batch_instance_id": "3c41837a-5727-11eb-9fe5-6e88dc41eb63"}}}} ```python -!argo delete seldon-batch-process +!argo delete -n default seldon-batch-process ``` Workflow 'seldon-batch-process' deleted - - -```python - -``` diff --git a/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/Chart.yaml b/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/Chart.yaml index bf39237512..e58516bcc5 100644 --- a/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/Chart.yaml +++ b/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/Chart.yaml @@ -6,4 +6,4 @@ keywords: name: seldon-batch-workflow sources: - https://github.com/SeldonIO/seldon-core -version: 0.1 +version: 0.2 diff --git a/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/templates/workflow.yaml b/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/templates/workflow.yaml index 798f5d74dd..dd587836eb 100644 --- a/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/templates/workflow.yaml +++ b/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/templates/workflow.yaml @@ -24,13 +24,17 @@ spec: resources: requests: storage: "{{ .Values.pvc.storage }}" + volumes: + - name: rclone-config + secret: + secretName: "{{ .Values.rclone.configSecretName }}" templates: - name: seldon-batch-process steps: - - - name: create-seldon-resource + - - name: create-seldon-resource template: create-seldon-resource-template - - name: wait-seldon-resource - template: wait-seldon-resource-template + template: wait-seldon-resource-template - - name: download-object-store template: download-object-store-template - - name: process-batch-inputs @@ -39,7 +43,7 @@ spec: template: upload-object-store-template - - name: delete-seldon-resource template: delete-seldon-resource-template - + - name: create-seldon-resource-template metadata: annotations: @@ -84,7 +88,7 @@ spec: name: classifier name: default replicas: {{ .Values.seldonDeployment.replicas }} - + - name: wait-seldon-resource-template metadata: annotations: @@ -96,34 +100,23 @@ spec: sleep {{ .Values.seldonDeployment.waitTime }} kubectl rollout status \ deploy/$(kubectl get deploy -l seldon-deployment-id="{{ .Values.seldonDeployment.name }}" -o jsonpath='{.items[0].metadata.name}') - + - name: download-object-store-template metadata: annotations: sidecar.istio.io/inject: "false" script: - image: minio/{{ .Values.minio.minioclient.image }} - env: - - name: MINIO_SERVER_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ .Values.minio.secret.name }} - key: {{ .Values.minio.secret.keyName.accesskey }} - - name: MINIO_SERVER_ACCESS_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.minio.secret.name }} - key: {{ .Values.minio.secret.keyName.secretkey }} - - name: MINIO_SERVER_HOST - value: {{ .Values.minio.endpoint }} + image: {{ .Values.rclone.image }} volumeMounts: - name: "{{ .Values.pvc.name }}" mountPath: /assets + - name: rclone-config + mountPath: /config/rclone + readOnly: true command: [sh] source: | - mc --insecure config host add minio-local $MINIO_SERVER_HOST $MINIO_SERVER_ACCESS_KEY $MINIO_SERVER_ACCESS_SECRET --api S3v4 - mc --insecure cp minio-local/{{ .Values.minio.inputDataPath }} /assets/input-data.txt - + rclone copyto {{ .Values.rclone.inputDataPath }} /assets/input-data.txt + - name: process-batch-inputs-template metadata: annotations: @@ -148,34 +141,24 @@ spec: --retries "{{ .Values.batchWorker.retries }}" \ --input-data-path "/assets/input-data.txt" \ --output-data-path "/assets/output-data.txt" - + - name: upload-object-store-template metadata: annotations: sidecar.istio.io/inject: "false" script: - image: minio/{{ .Values.minio.minioclient.image }} + image: {{ .Values.rclone.image }} volumeMounts: - name: "{{ .Values.pvc.name }}" mountPath: /assets + - name: rclone-config + mountPath: /config/rclone + readOnly: true command: [sh] - env: - - name: MINIO_SERVER_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ .Values.minio.secret.name }} - key: {{ .Values.minio.secret.keyName.accesskey }} - - name: MINIO_SERVER_ACCESS_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.minio.secret.name }} - key: {{ .Values.minio.secret.keyName.secretkey }} - - name: MINIO_SERVER_HOST - value: {{ .Values.minio.endpoint }} source: | - mc --insecure config host add minio-local $MINIO_SERVER_HOST $MINIO_SERVER_ACCESS_KEY $MINIO_SERVER_ACCESS_SECRET --api S3v4 - mc --insecure cp /assets/output-data.txt minio-local/{{ .Values.minio.outputDataPath }} - + rclone copyto /assets/output-data.txt {{ .Values.rclone.outputDataPath }} + + - name: delete-seldon-resource-template metadata: annotations: @@ -186,4 +169,3 @@ spec: source: | sleep {{ .Values.seldonDeployment.waitTime }} kubectl delete seldondeployments -n {{ .Values.workflow.namespace }} {{ .Values.seldonDeployment.name }} - diff --git a/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/values.yaml b/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/values.yaml index 69da0d1a20..3820c36ea5 100644 --- a/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/values.yaml +++ b/examples/batch/argo-workflows-batch/helm-charts/seldon-batch-workflow/values.yaml @@ -8,7 +8,7 @@ workflow: pvc: # Name of the persistent volume claim to be created name: seldon-pvc - # Size of the storage volume to be created for the batch job + # Size of the storage volume to be created for the batch job storage: 2Mi # Seldon deployment to be created for batch processing seldonDeployment: @@ -54,22 +54,12 @@ batchWorker: dataType: "data" # Whether to enable benchmarking on the batch processor worker enableBenchmark: true -minio: - # Minio Client (mc) Image name - minioclient: - image: mc:RELEASE.2020-10-03T02-54-56Z - # The location of the minio endpoint - endpoint: http://minio.minio-system.svc.cluster.local:9000 - # This is the secret that should contain the values to access minio - secret: - # The name of the secret which by default is "minio" but you can create a different one - name: minio - keyName: - # The key name inside that secret to find the access key to authenticate minio - accesskey: accesskey - # The key name inside that secret to find the secret key to authenticate minio - secretkey: secretkey - # The name of the file inside of minio that will contain the batch data to process - inputDataPath: data/input-data.txt - # The name of the file inside of minio that will contain the batch data to process - outputDataPath: data/output-data-{{workflow.uid}}.txt +rclone: + # Rclone Image Name + image: rclone/rclone:1.53 + # Name of secret containing rclone.conf file + configSecretName: rclone-config-secret + # The source:sourcepath of file for rclone to pull that will contain the batch data to process + inputDataPath: cluster-minio:data/input-data.txt + # The dest:destpath for rclone to know where to push results of batch processing + outputDataPath: cluster-minio:data/output-data-{{workflow.uid}}.txt diff --git a/examples/batch/argo-workflows-batch/rclone-config.yaml b/examples/batch/argo-workflows-batch/rclone-config.yaml new file mode 100644 index 0000000000..c6f60aea64 --- /dev/null +++ b/examples/batch/argo-workflows-batch/rclone-config.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Secret +metadata: + name: rclone-config-secret +type: Opaque +stringData: + rclone.conf: | + [cluster-minio] + type = s3 + provider = minio + env_auth = false + access_key_id = minioadmin + secret_access_key = minioadmin + endpoint = http://minio.minio-system.svc.cluster.local:9000 diff --git a/examples/batch/argo-workflows-batch/role.yaml b/examples/batch/argo-workflows-batch/role.yaml new file mode 100644 index 0000000000..dbbd791cff --- /dev/null +++ b/examples/batch/argo-workflows-batch/role.yaml @@ -0,0 +1,29 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: workflow +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - "*" +- apiGroups: + - "apps" + resources: + - deployments + verbs: + - "*" +- apiGroups: + - "" + resources: + - pods/log + verbs: + - "*" +- apiGroups: + - machinelearning.seldon.io + resources: + - "*" + verbs: + - "*"