root@Devin-ThinkStation:/mnt/e/git/AML-Kubernetes/examples/inference/simple-flow# kubectl get pods -A NAMESPACE NAME READY STATUS RESTARTS AGE azure-arc cluster-metadata-operator-5968988cbc-6cnz4 2/2 Running 4 (176m ago) 4d4h azure-arc clusterconnect-agent-546f884d7-494dv 3/3 Running 6 (176m ago) 4d4h azure-arc clusteridentityoperator-b4ffb57b4-jbdl8 2/2 Running 5 (176m ago) 4d4h azure-arc config-agent-78d69bc9bc-dqqsw 2/2 Running 5 (176m ago) 4d4h azure-arc controller-manager-77f4777bc9-qbwgj 2/2 Running 5 (176m ago) 4d4h azure-arc extension-manager-86bf565696-rnmk5 2/2 Running 5 (176m ago) 4d4h azure-arc flux-logs-agent-7457dbf6df-cjf5b 1/1 Running 2 (176m ago) 4d4h azure-arc kube-aad-proxy-5cd947df6-kqbn4 2/2 Running 6 (175m ago) 4d4h azure-arc metrics-agent-69c47c6459-7m6zh 2/2 Running 4 (176m ago) 4d4h azure-arc resource-sync-agent-55589c88b8-mzdkr 2/2 Running 4 (176m ago) 4d4h azureml aml-operator-6674644b7f-vqsbr 2/2 Running 4 (176m ago) 4d3h azureml amlarc-identity-controller-7b9786ccd-vgl7x 1/1 Running 2 (176m ago) 4d3h azureml amlarc-identity-proxy-689f54f95-xbm7v 1/1 Running 2 (176m ago) 4d3h azureml azureml-fe-v2-7d948b67fb-d55mz 3/3 Running 6 (176m ago) 4d3h azureml crbs-stable-a085d67d-deployment-74b8d96597-97rg5 1/1 Running 2 (176m ago) 5h azureml devinarcmlext-kube-state-metrics-7d86c9479d-c45tv 1/1 Running 2 (176m ago) 4d3h azureml devinarcmlext-prometheus-operator-679b8bbd58-zdfc2 1/1 Running 2 (176m ago) 4d3h azureml fluent-bit-6ftgp 1/1 Running 2 (176m ago) 4d3h azureml gateway-74c67c76cc-g586b 1/1 Running 2 (176m ago) 4d3h azureml healthcheck 0/1 Completed 0 173m azureml inference-operator-controller-manager-6759c7bcd5-pctmt 1/1 Running 3 (176m ago) 4d3h azureml metrics-controller-manager-6484965cd9-gw2cg 1/1 Running 2 (176m ago) 4d3h azureml prometheus-prom-prometheus-0 2/2 Running 4 (176m ago) 5h azureml relayserver-db9bd66b6-d9trg 1/1 Running 5 (175m ago) 4d3h azureml relayserver-db9bd66b6-glw25 1/1 Running 5 (175m ago) 4d3h azureml volcano-admission-6cdcd5f487-b5b6p 1/1 Running 2 (176m ago) 4d3h azureml volcano-controllers-79fbdddc9-w6b97 1/1 Running 2 (176m ago) 4d3h azureml volcano-scheduler-cf476658c-mrxjk 1/1 Running 2 (176m ago) 4d3h default blue-sklearn-mnist-minikube-58f5fd4cf7-lgvmf 2/2 Running 0 61m kube-system coredns-565d847f94-gtplj 1/1 Running 2 (176m ago) 4d4h kube-system etcd-minikube 1/1 Running 2 (176m ago) 4d4h kube-system kube-apiserver-minikube 1/1 Running 2 (176m ago) 4d4h kube-system kube-controller-manager-minikube 1/1 Running 2 (176m ago) 4d4h kube-system kube-proxy-p6psw 1/1 Running 2 (176m ago) 4d4h kube-system kube-scheduler-minikube 1/1 Running 2 (176m ago) 4d4h kube-system storage-provisioner 1/1 Running 4 (176m ago) 4d4h mdc microsoft-defender-collectors-ccb8k 3/3 Running 6 (176m ago) 4d4h mdc microsoft-defender-pod-collector-misc-84fc8b69f4-9mn7h 1/1 Running 2 (176m ago) 4d4h mdc microsoft-defender-pod-collector-virtual-kubelet-69b69894dm496x 1/1 Running 2 (176m ago) 4d4h mdc microsoft-defender-publisher-fqfdp 1/1 Running 2 (176m ago) 4d4h =========================================================================================================== root@Devin-ThinkStation:/mnt/e/git/AML-Kubernetes/examples/inference/simple-flow# kubectl logs blue-sklearn-mnist-minikube-58f5fd4cf7-lgvmf Defaulted container "inference-server" out of: inference-server, identity-sidecar, storageinitializer-modeldata (init) 2022-12-12T23:54:50,916758900+00:00 - iot-server/run 2022-12-12T23:54:50,921648600+00:00 - nginx/run 2022-12-12T23:54:50,923256800+00:00 - gunicorn/run 2022-12-12T23:54:50,923645800+00:00 - rsyslog/run 2022-12-12T23:54:50,924279700+00:00 | gunicorn/run | 2022-12-12T23:54:50,925067800+00:00 | gunicorn/run | ############################################### 2022-12-12T23:54:50,925874800+00:00 | gunicorn/run | AzureML Container Runtime Information 2022-12-12T23:54:50,926678900+00:00 | gunicorn/run | ############################################### 2022-12-12T23:54:51,221757000+00:00 | gunicorn/run | 2022-12-12T23:54:51,223103500+00:00 | gunicorn/run | 2022-12-12T23:54:51,225338300+00:00 | gunicorn/run | AzureML image information: openmpi3.1.2-ubuntu18.04, Materializaton Build:20221129.v2 2022-12-12T23:54:51,226392200+00:00 | gunicorn/run | 2022-12-12T23:54:51,227566300+00:00 | gunicorn/run | 2022-12-12T23:54:51,228571700+00:00 | gunicorn/run | PATH environment variable: /azureml-envs/azureml_eb47cb56bd00bf9ef74ac887556e6bde/bin:/opt/miniconda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 2022-12-12T23:54:51,321915100+00:00 | gunicorn/run | PYTHONPATH environment variable: 2022-12-12T23:54:51,322699900+00:00 | gunicorn/run | 2022-12-12T23:54:51,323568700+00:00 | gunicorn/run | Pip Dependencies (before dynamic installation) EdgeHubConnectionString and IOTEDGE_IOTHUBHOSTNAME are not set. Exiting... 2022-12-12T23:54:51,725690200+00:00 - iot-server/finish 1 0 2022-12-12T23:54:51,726482800+00:00 - Exit code 1 is normal. Not restarting iot-server. adal==1.2.7 applicationinsights==0.11.10 argcomplete==1.12.3 azure-common==1.1.28 azure-core==1.21.1 azure-graphrbac==0.61.1 azure-identity==1.7.0 azure-mgmt-authorization==0.61.0 azure-mgmt-containerregistry==8.2.0 azure-mgmt-core==1.3.0 azure-mgmt-keyvault==9.3.0 azure-mgmt-resource==20.1.0 azure-mgmt-storage==19.1.0 azureml-core==1.38.0.post2 azureml-dataprep==2.26.0 azureml-dataprep-native==38.0.0 azureml-dataprep-rslex==2.2.0 azureml-dataset-runtime==1.38.0 azureml-defaults==1.38.0 azureml-inference-server-http==0.4.14 backports.tempfile==1.0 backports.weakref==1.0.post1 bcrypt==4.0.1 cachetools==5.2.0 certifi==2022.9.24 cffi==1.15.1 charset-normalizer==2.1.1 click==7.1.2 cloudpickle==2.2.0 colorama==0.4.6 configparser==3.7.4 contextlib2==21.6.0 cryptography==36.0.2 distro==1.8.0 docker==5.0.3 dotnetcore2==2.1.23 Flask==1.0.3 fusepy==3.0.1 google-api-core==2.11.0 google-auth==2.15.0 googleapis-common-protos==1.57.0 gunicorn==20.1.0 humanfriendly==10.0 idna==3.4 importlib-metadata==4.13.0 inference-schema==1.3.0 isodate==0.6.1 itsdangerous==1.1.0 jeepney==0.8.0 Jinja2==3.0.3 jmespath==0.10.0 joblib==1.0.1 json-logging-py==0.2 jsonpickle==2.2.0 knack==0.8.2 MarkupSafe==2.1.1 msal==1.20.0 msal-extensions==0.3.1 msrest==0.6.21 msrestazure==0.6.4 ndg-httpsclient==0.5.1 numpy @ file:///home/conda/feedstock_root/build_artifacts/numpy_1629092064233/work oauthlib==3.2.2 opencensus==0.11.0 opencensus-context==0.1.3 opencensus-ext-azure==1.1.7 packaging==21.3 paramiko==2.12.0 pathspec==0.10.2 pkginfo==1.9.2 portalocker==2.6.0 protobuf==4.21.10 psutil==5.9.4 pyarrow==3.0.0 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.21 Pygments==2.13.0 PyJWT==2.6.0 PyNaCl==1.5.0 pyOpenSSL==21.0.0 pyparsing==3.0.9 PySocks==1.7.1 python-dateutil==2.8.2 pytz==2022.6 PyYAML==6.0 requests==2.28.1 requests-oauthlib==1.3.1 rsa==4.9 scikit-learn @ file:///home/conda/feedstock_root/build_artifacts/scikit-learn_1630910536896/work scipy @ file:///home/conda/feedstock_root/build_artifacts/scipy_1628206376058/work SecretStorage==3.3.3 six==1.16.0 tabulate==0.9.0 threadpoolctl @ file:///home/conda/feedstock_root/build_artifacts/threadpoolctl_1643647933166/work typing_extensions==4.4.0 urllib3==1.26.7 websocket-client==1.4.2 Werkzeug==1.0.1 wrapt==1.12.1 zipp==3.11.0 2022-12-12T23:54:53,822456100+00:00 | gunicorn/run | 2022-12-12T23:54:53,824031000+00:00 | gunicorn/run | ############################################### 2022-12-12T23:54:53,825795700+00:00 | gunicorn/run | AzureML Inference Server 2022-12-12T23:54:53,826958500+00:00 | gunicorn/run | ############################################### 2022-12-12T23:54:53,827977700+00:00 | gunicorn/run | 2022-12-12T23:54:56,425633100+00:00 | gunicorn/run | Starting AzureML Inference Server HTTP. Valid Application Insights instrumentation key provided. Azure ML Inferencing HTTP server v0.4.14 Server Settings --------------- Entry Script Name: score.py Model Directory: /var/azureml-app/azureml-models/18f10a82cef1a8fb046c96992a72234b/1 Worker Count: 1 Worker Timeout (seconds): 300 Server Port: 31311 Application Insights Enabled: true Application Insights Key: AppInsights key provided Server Routes --------------- Liveness Probe: GET 127.0.0.1:31311/ Score: POST 127.0.0.1:31311/score Starting gunicorn 20.1.0 Listening at: http://0.0.0.0:31311 (11) Using worker: sync Booting worker with pid: 85 Initializing logger 2022-12-12 23:54:58,429 | root | INFO | Starting up app insights client logging socket was found. logging is available. logging socket was found. logging is available. 2022-12-12 23:54:58,429 | root | INFO | Starting up request id generator 2022-12-12 23:54:58,429 | root | INFO | Starting up app insight hooks 2022-12-12 23:54:58,429 | root | INFO | Invoking user's init function no request id,/azureml-envs/azureml_eb47cb56bd00bf9ef74ac887556e6bde/lib/python3.7/site-packages/sklearn/base.py:315: UserWarning: Trying to unpickle estimator LogisticRegression from version 0.22.1 when using version 0.24.2. This might lead to breaking code or invalid results. Use at your own risk. UserWarning) 2022-12-12 23:54:59,432 | root | INFO | Users's init has completed successfully /azureml-envs/azureml_eb47cb56bd00bf9ef74ac887556e6bde/lib/python3.7/site-packages/sklearn/base.py:315: UserWarning: Trying to unpickle estimator LogisticRegression from version 0.22.1 when using version 0.24.2. This might lead to breaking code or invalid results. Use at your own risk. UserWarning) 2022-12-12 23:54:59,433 | root | INFO | Skipping middleware: dbg_model_info as it's not enabled. 2022-12-12 23:54:59,433 | root | INFO | Skipping middleware: dbg_resource_usage as it's not enabled. 2022-12-12 23:54:59,434 | root | INFO | Scoring timeout setting is not found. Use default timeout: 3600000 ms root@Devin-ThinkStation:/mnt/e/git/AML-Kubernetes/examples/inference/simple-flow# az ml online-deployment get-logs --name blue --endpoint sklearn-mnist ^Caz ml online-endpoint show -n sklearn-mnistroot@Devin-ThinkStation:/mnt/e/git/AML-Kubernetes/examples/inference/simple-flow# az ml online-endpoint show -n sklearn-mnist-minikube { "auth_mode": "key", "compute": "azureml:/subscriptions/0d13d072-64ed-4c78-b54a-31a7fab3ad01/resourceGroups/devinARC-ML-RG/providers/Microsoft.MachineLearningServices/workspaces/devinWS/computes/devinLocCmpt", "id": "/subscriptions/0d13d072-64ed-4c78-b54a-31a7fab3ad01/resourceGroups/devinARC-ML-RG/providers/Microsoft.MachineLearningServices/workspaces/devinWS/onlineEndpoints/sklearn-mnist-minikube", "identity": { "principal_id": "4f7d602f-ea27-4501-b2c7-b85b4c3f4d00", "tenant_id": "72f988bf-86f1-41af-91ab-2d7cd011db47", "type": "system_assigned" }, "kind": "K8S", "location": "eastus2", "mirror_traffic": {}, "name": "sklearn-mnist-minikube", "openapi_uri": "http://10.106.30.249/api/v1/endpoint/sklearn-mnist-minikube/swagger.json", "properties": { "AzureAsyncOperationUri": "https://management.azure.com/subscriptions/0d13d072-64ed-4c78-b54a-31a7fab3ad01/providers/Microsoft.MachineLearningServices/locations/eastus2/mfeOperationsStatus/oe:36078857-ed80-438a-815c-f20e476be657:b04738dc-3dda-4198-8b14-1477da0cd73a?api-version=2022-02-01-preview", "azureml.onlineendpointid": "/subscriptions/0d13d072-64ed-4c78-b54a-31a7fab3ad01/resourcegroups/devinarc-ml-rg/providers/microsoft.machinelearningservices/workspaces/devinws/onlineendpoints/sklearn-mnist-minikube" }, "provisioning_state": "Succeeded", "resourceGroup": "devinARC-ML-RG", "scoring_uri": "http://10.106.30.249/api/v1/endpoint/sklearn-mnist-minikube/score", "tags": {}, "traffic": { "blue": 100 } } ======================================================= root@Devin-ThinkStation:/mnt/e/git/AML-Kubernetes/examples/inference/simple-flow# az ml online-endpoint invoke -n sklearn-mnist-minikube -r sample-request.json (, 'Connection to 10.106.30.249 timed out. (connect timeout=300)')