diff --git a/examples/models/mlflow_server_ab_test_ambassador/MLproject b/examples/models/mlflow_server_ab_test_ambassador/MLproject new file mode 100644 index 0000000000..ade1378bc2 --- /dev/null +++ b/examples/models/mlflow_server_ab_test_ambassador/MLproject @@ -0,0 +1,10 @@ +name: mlflow-talk + +conda_env: conda.yaml + +entry_points: + main: + parameters: + alpha: float + l1_ratio: {type: float, default: 0.1} + command: "python train.py {alpha} {l1_ratio}" diff --git a/examples/models/mlflow_server_ab_test_ambassador/ab-test-mlflow-model-server-seldon-config.yaml b/examples/models/mlflow_server_ab_test_ambassador/ab-test-mlflow-model-server-seldon-config.yaml index b178aa777a..e5733b2a10 100644 --- a/examples/models/mlflow_server_ab_test_ambassador/ab-test-mlflow-model-server-seldon-config.yaml +++ b/examples/models/mlflow_server_ab_test_ambassador/ab-test-mlflow-model-server-seldon-config.yaml @@ -6,19 +6,19 @@ metadata: spec: name: mlflow-deployment predictors: - - graph: - children: [] - implementation: MLFLOW_SERVER - modelUri: gs://seldon-models/mlflow/elasticnet_wine - name: wines-classifier - name: a-mlflow-deployment-dag - replicas: 1 - traffic: 20 - - graph: - children: [] - implementation: MLFLOW_SERVER - modelUri: gs://seldon-models/mlflow/elasticnet_wine - name: wines-classifier - name: b-mlflow-deployment-dag - replicas: 1 - traffic: 80 + - graph: + children: [] + implementation: MLFLOW_SERVER + modelUri: gs://seldon-models/mlflow/model-a + name: wines-classifier + name: a-mlflow-deployment-dag + replicas: 1 + traffic: 80 + - graph: + children: [] + implementation: MLFLOW_SERVER + modelUri: gs://seldon-models/mlflow/model-b + name: wines-classifier + name: b-mlflow-deployment-dag + replicas: 1 + traffic: 20 diff --git a/examples/models/mlflow_server_ab_test_ambassador/conda.yaml b/examples/models/mlflow_server_ab_test_ambassador/conda.yaml new file mode 100644 index 0000000000..050bc82be3 --- /dev/null +++ b/examples/models/mlflow_server_ab_test_ambassador/conda.yaml @@ -0,0 +1,8 @@ +name: mlflow-talk +channels: + - defaults +dependencies: + - python=3.6 + - scikit-learn=0.19.1 + - pip: + - mlflow>=1.0 diff --git a/examples/models/mlflow_server_ab_test_ambassador/images/grafana-mlflow.jpg b/examples/models/mlflow_server_ab_test_ambassador/images/grafana-mlflow.jpg old mode 100755 new mode 100644 index a22926924e..0802fe4796 Binary files a/examples/models/mlflow_server_ab_test_ambassador/images/grafana-mlflow.jpg and b/examples/models/mlflow_server_ab_test_ambassador/images/grafana-mlflow.jpg differ diff --git a/examples/models/mlflow_server_ab_test_ambassador/mlflow-model-server-seldon-config.yaml b/examples/models/mlflow_server_ab_test_ambassador/mlflow-model-server-seldon-config.yaml index 9a6440cc34..3ab59b801f 100644 --- a/examples/models/mlflow_server_ab_test_ambassador/mlflow-model-server-seldon-config.yaml +++ b/examples/models/mlflow_server_ab_test_ambassador/mlflow-model-server-seldon-config.yaml @@ -6,10 +6,10 @@ metadata: spec: name: mlflow-deployment predictors: - - graph: - children: [] - implementation: MLFLOW_SERVER - modelUri: gs://seldon-models/mlflow/elasticnet_wine - name: wines-classifier - name: mlflow-deployment-dag - replicas: 1 + - graph: + children: [] + implementation: MLFLOW_SERVER + modelUri: gs://seldon-models/mlflow/model-a + name: wines-classifier + name: mlflow-deployment-dag + replicas: 1 diff --git a/examples/models/mlflow_server_ab_test_ambassador/mlflow_server_ab_test_ambassador.ipynb b/examples/models/mlflow_server_ab_test_ambassador/mlflow_server_ab_test_ambassador.ipynb index f7c3126ee7..1f7b1e3d8a 100644 --- a/examples/models/mlflow_server_ab_test_ambassador/mlflow_server_ab_test_ambassador.ipynb +++ b/examples/models/mlflow_server_ab_test_ambassador/mlflow_server_ab_test_ambassador.ipynb @@ -5,28 +5,38 @@ "metadata": {}, "source": [ "# MLFlow Pre-packaged Model Server AB Test Deployment \n", - "In this example we will build two models with MLFlow and we will deploy them as an A/B test deployment.\n", - "\n", - "The reason this is powerful is because it allows you to deploy a new model next to the old one, distributing a percentage of traffic.\n", - "\n", - "These deployment strategies are quite simple using Seldon, and can be extended to shadow deployments, multi-armed-bandits, etc.\n", - "\n", - "\n", + "In this example we will build two models with MLFlow and we will deploy them as an A/B test deployment. The reason this is powerful is because it allows you to deploy a new model next to the old one, distributing a percentage of traffic. These deployment strategies are quite simple using Seldon, and can be extended to shadow deployments, multi-armed-bandits, etc." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "## Tutorial Overview\n", "\n", - "This tutorial will break down in the following sections:\n", + "This tutorial will follow closely break down in the following sections:\n", "\n", - "1) Train the MLFlow elastic net wine example\n", + "1. Train the MLFlow elastic net wine example\n", "\n", - "2) Deploy your trained model leveraging our pre-packaged MLFlow model server\n", + "2. Deploy your trained model leveraging our pre-packaged MLFlow model server\n", "\n", - "3) Test the deployed MLFlow model by sending requests\n", + "3. Test the deployed MLFlow model by sending requests\n", "\n", - "4) Deploy your second model as an A/B test\n", + "4. Deploy your second model as an A/B test\n", "\n", - "5) Visualise and monitor the performance of your models using Seldon Analytics\n", + "5. Visualise and monitor the performance of your models using Seldon Analytics\n", "\n", - "## Dependencies:\n", + "It will follow closely our talk at the [Spark + AI Summit 2019 on Seldon and MLflow](https://www.youtube.com/watch?v=D6eSfd9w9eA)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "toc-hr-collapsed": true, + "toc-nb-collapsed": true + }, + "source": [ + "## Dependencies\n", "\n", "For this example to work you must be running Seldon 0.3.2 or above - you can follow our [getting started guide for this](https://docs.seldon.io/projects/seldon-core/en/latest/workflow/install.html).\n", "\n", @@ -36,46 +46,212 @@ "* kubectl v1.14+\n", "* Python 3.6+\n", "* MLFlow 1.1.0\n", + "* pygmentize\n", + "\n", + "We will also take this chance to load the Python dependencies we will use through the tutorial:" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "from seldon_core.seldon_client import SeldonClient" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "toc-hr-collapsed": true, + "toc-nb-collapsed": true + }, + "source": [ + "#### Let's get started! 🚀🔥" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "toc-hr-collapsed": true, + "toc-nb-collapsed": true + }, + "source": [ + "## 1. Train the first MLFlow Elastic Net Wine example\n", "\n", - "#### Let's get started! 🚀🔥\n" + "For our example, we will use the elastic net wine example from [MLflow's tutorial](https://www.mlflow.org/docs/latest/tutorial.html)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## 1) Train the first MLFlow Elastic Net Wine example\n", - "We will use the elastic net wine example from MLFlow v1.1.0 for this example. First we'll import all the dependencies." + "### MLproject\n", + "\n", + "As any other MLflow project, it is defined by its `MLproject` file:" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[94mname\u001b[39;49;00m: mlflow-talk\n", + "\n", + "\u001b[94mconda_env\u001b[39;49;00m: conda.yaml\n", + "\n", + "\u001b[94mentry_points\u001b[39;49;00m:\n", + " \u001b[94mmain\u001b[39;49;00m:\n", + " \u001b[94mparameters\u001b[39;49;00m:\n", + " \u001b[94malpha\u001b[39;49;00m: float\n", + " \u001b[94ml1_ratio\u001b[39;49;00m: {\u001b[94mtype\u001b[39;49;00m: \u001b[31mfloat\u001b[39;49;00m,\u001b[94m default\u001b[39;49;00m: \u001b[31m0.1\u001b[39;49;00m}\n", + " \u001b[94mcommand\u001b[39;49;00m: \u001b[33m\"\u001b[39;49;00m\u001b[33mpython\u001b[39;49;00m\u001b[31m \u001b[39;49;00m\u001b[33mtrain.py\u001b[39;49;00m\u001b[31m \u001b[39;49;00m\u001b[33m{alpha}\u001b[39;49;00m\u001b[31m \u001b[39;49;00m\u001b[33m{l1_ratio}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\n" + ] + } + ], "source": [ - "import os, warnings, sys\n", - "\n", - "import pandas as pd\n", - "import numpy as np\n", - "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.linear_model import ElasticNet\n", - "\n", - "import mlflow\n", - "import mlflow.sklearn" + "!pygmentize -l yaml MLproject" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Let's load the wine dataset which is also in this folder" + "We can see that this project uses Conda for the environment and that it's defined in the `conda.yaml` file:" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[94mname\u001b[39;49;00m: mlflow-talk\n", + "\u001b[94mchannels\u001b[39;49;00m:\n", + " - defaults\n", + "\u001b[94mdependencies\u001b[39;49;00m:\n", + " - python=3.6\n", + " - scikit-learn=0.19.1\n", + " - \u001b[94mpip\u001b[39;49;00m:\n", + " - mlflow>=1.0\n" + ] + } + ], + "source": [ + "!pygmentize conda.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Lastly, we can also see that the training will be performed by the `train.py` file, which receives two parameters `alpha` and `l1_ratio`:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[37m# The data set used in this example is from http://archive.ics.uci.edu/ml/datasets/Wine+Quality\u001b[39;49;00m\n", + "\u001b[37m# P. Cortez, A. Cerdeira, F. Almeida, T. Matos and J. Reis.\u001b[39;49;00m\n", + "\u001b[37m# Modeling wine preferences by data mining from physicochemical properties. In Decision Support Systems, Elsevier, 47(4):547-553, 2009.\u001b[39;49;00m\n", + "\n", + "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mos\u001b[39;49;00m\n", + "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mwarnings\u001b[39;49;00m\n", + "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36msys\u001b[39;49;00m\n", + "\n", + "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mpandas\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mpd\u001b[39;49;00m\n", + "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mnumpy\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mnp\u001b[39;49;00m\n", + "\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36msklearn.metrics\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m mean_squared_error, mean_absolute_error, r2_score\n", + "\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36msklearn.model_selection\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m train_test_split\n", + "\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36msklearn.linear_model\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m ElasticNet\n", + "\n", + "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mmlflow\u001b[39;49;00m\n", + "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mmlflow.sklearn\u001b[39;49;00m\n", + "\n", + "\n", + "\u001b[34mdef\u001b[39;49;00m \u001b[32meval_metrics\u001b[39;49;00m(actual, pred):\n", + " rmse = np.sqrt(mean_squared_error(actual, pred))\n", + " mae = mean_absolute_error(actual, pred)\n", + " r2 = r2_score(actual, pred)\n", + " \u001b[34mreturn\u001b[39;49;00m rmse, mae, r2\n", + "\n", + "\n", + "\n", + "\u001b[34mif\u001b[39;49;00m \u001b[31m__name__\u001b[39;49;00m == \u001b[33m\"\u001b[39;49;00m\u001b[33m__main__\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\n", + " warnings.filterwarnings(\u001b[33m\"\u001b[39;49;00m\u001b[33mignore\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n", + " np.random.seed(\u001b[34m40\u001b[39;49;00m)\n", + "\n", + " \u001b[37m# Read the wine-quality csv file (make sure you're running this from the root of MLflow!)\u001b[39;49;00m\n", + " wine_path = os.path.join(os.path.dirname(os.path.abspath(\u001b[31m__file__\u001b[39;49;00m)), \u001b[33m\"\u001b[39;49;00m\u001b[33mwine-quality.csv\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n", + " data = pd.read_csv(wine_path)\n", + "\n", + " \u001b[37m# Split the data into training and test sets. (0.75, 0.25) split.\u001b[39;49;00m\n", + " train, test = train_test_split(data)\n", + "\n", + " \u001b[37m# The predicted column is \"quality\" which is a scalar from [3, 9]\u001b[39;49;00m\n", + " train_x = train.drop([\u001b[33m\"\u001b[39;49;00m\u001b[33mquality\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m], axis=\u001b[34m1\u001b[39;49;00m)\n", + " test_x = test.drop([\u001b[33m\"\u001b[39;49;00m\u001b[33mquality\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m], axis=\u001b[34m1\u001b[39;49;00m)\n", + " train_y = train[[\u001b[33m\"\u001b[39;49;00m\u001b[33mquality\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m]]\n", + " test_y = test[[\u001b[33m\"\u001b[39;49;00m\u001b[33mquality\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m]]\n", + "\n", + " alpha = \u001b[36mfloat\u001b[39;49;00m(sys.argv[\u001b[34m1\u001b[39;49;00m]) \u001b[34mif\u001b[39;49;00m \u001b[36mlen\u001b[39;49;00m(sys.argv) > \u001b[34m1\u001b[39;49;00m \u001b[34melse\u001b[39;49;00m \u001b[34m0.5\u001b[39;49;00m\n", + " l1_ratio = \u001b[36mfloat\u001b[39;49;00m(sys.argv[\u001b[34m2\u001b[39;49;00m]) \u001b[34mif\u001b[39;49;00m \u001b[36mlen\u001b[39;49;00m(sys.argv) > \u001b[34m2\u001b[39;49;00m \u001b[34melse\u001b[39;49;00m \u001b[34m0.5\u001b[39;49;00m\n", + "\n", + " \u001b[34mwith\u001b[39;49;00m mlflow.start_run():\n", + " lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=\u001b[34m42\u001b[39;49;00m)\n", + " lr.fit(train_x, train_y)\n", + "\n", + " predicted_qualities = lr.predict(test_x)\n", + "\n", + " (rmse, mae, r2) = eval_metrics(test_y, predicted_qualities)\n", + "\n", + " \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33mElasticnet model (alpha=\u001b[39;49;00m\u001b[33m%f\u001b[39;49;00m\u001b[33m, l1_ratio=\u001b[39;49;00m\u001b[33m%f\u001b[39;49;00m\u001b[33m):\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m % (alpha, l1_ratio))\n", + " \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33m RMSE: \u001b[39;49;00m\u001b[33m%s\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m % rmse)\n", + " \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33m MAE: \u001b[39;49;00m\u001b[33m%s\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m % mae)\n", + " \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33m R2: \u001b[39;49;00m\u001b[33m%s\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m % r2)\n", + "\n", + " mlflow.log_param(\u001b[33m\"\u001b[39;49;00m\u001b[33malpha\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m, alpha)\n", + " mlflow.log_param(\u001b[33m\"\u001b[39;49;00m\u001b[33ml1_ratio\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m, l1_ratio)\n", + " mlflow.log_metric(\u001b[33m\"\u001b[39;49;00m\u001b[33mrmse\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m, rmse)\n", + " mlflow.log_metric(\u001b[33m\"\u001b[39;49;00m\u001b[33mr2\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m, r2)\n", + " mlflow.log_metric(\u001b[33m\"\u001b[39;49;00m\u001b[33mmae\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m, mae)\n", + "\n", + " mlflow.sklearn.log_model(lr, \u001b[33m\"\u001b[39;49;00m\u001b[33mmodel\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n" + ] + } + ], + "source": [ + "!pygmentize train.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dataset\n", + "\n", + "We will use the wine quality dataset.\n", + "Let's load it to see what's inside:" + ] + }, + { + "cell_type": "code", + "execution_count": 53, "metadata": {}, "outputs": [ { @@ -216,7 +392,7 @@ "4 9.9 6 " ] }, - "execution_count": 3, + "execution_count": 53, "metadata": {}, "output_type": "execute_result" } @@ -230,152 +406,189 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Then we will define all the functions we will use to train the model" + "### Training\n", + "\n", + "We've set up our MLflow project and our dataset is ready, so we are now good to start training.\n", + "MLflow allows us to train our model with the following command:\n", + "\n", + "``` bash\n", + "$ mlflow run . -P alpha=... -P l1_ratio=...\n", + "```\n", + "\n", + "On each run, `mlflow` will set up the Conda environment defined by the `conda.yaml` file and will run the training commands defined in the `MLproject` file." ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2019/11/20 11:16:37 INFO mlflow.projects: === Created directory /tmp/tmpaok27ecp for downloading remote URIs passed to arguments of type 'path' ===\n", + "2019/11/20 11:16:37 INFO mlflow.projects: === Running command 'source /opt/miniconda3/bin/../etc/profile.d/conda.sh && conda activate mlflow-1ecba04797edb7e7f7212d429debd9b664c31651 1>&2 && python train.py 0.5 0.5' in run with ID 'fbbb1fe4f9ef4b4faf370f8a946f7c60' === \n", + "Elasticnet model (alpha=0.500000, l1_ratio=0.500000):\n", + " RMSE: 0.82224284975954\n", + " MAE: 0.6278761410160691\n", + " R2: 0.12678721972772689\n", + "2019/11/20 11:16:38 INFO mlflow.projects: === Run (ID 'fbbb1fe4f9ef4b4faf370f8a946f7c60') succeeded ===\n" + ] + } + ], "source": [ - "def eval_metrics(actual, pred):\n", - " rmse = np.sqrt(mean_squared_error(actual, pred))\n", - " mae = mean_absolute_error(actual, pred)\n", - " r2 = r2_score(actual, pred)\n", - " return rmse, mae, r2\n", - "\n", - "def run_train_model_iteration(data, seed=40):\n", - " \"\"\"\n", - " This function takes a pandas dataframe and returns \n", - " \"\"\"\n", - " np.random.seed(seed)\n", - " train, test = train_test_split(data)\n", - " train_x = train.drop([\"quality\"], axis=1)\n", - " test_x = test.drop([\"quality\"], axis=1)\n", - " train_y = train[[\"quality\"]]\n", - " test_y = test[[\"quality\"]]\n", - "\n", - " alpha = 0.5\n", - " l1_ratio = 0.5\n", - "\n", - " with mlflow.start_run():\n", - " lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42)\n", - " lr.fit(train_x, train_y)\n", - "\n", - " predicted_qualities = lr.predict(test_x)\n", - "\n", - " (rmse, mae, r2) = eval_metrics(test_y, predicted_qualities)\n", + "!mlflow run . -P alpha=0.5 -P l1_ratio=0.5" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Each of these commands will create a new run which can be visualised through the MLFlow dashboard as per the screenshot below.\n", "\n", - " print(\"Elasticnet model (alpha=%f, l1_ratio=%f):\" % (alpha, l1_ratio))\n", - " print(\" RMSE: %s\" % rmse)\n", - " print(\" MAE: %s\" % mae)\n", - " print(\" R2: %s\" % r2)\n", - " \n", - " # We create a store of the model \n", - " mlflow.sklearn.log_model(lr, \"model\")\n" + "![](images/mlflow-dashboard.png)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Now we can create a first trained model, which the function above creates an MLFlow \"log\"" + "Each of these models can actually be found on the `mlruns` folder:" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Elasticnet model (alpha=0.500000, l1_ratio=0.500000):\n", - " RMSE: 0.8222428497595403\n", - " MAE: 0.6278761410160693\n", - " R2: 0.12678721972772622\n" + "\u001b[01;34mmlruns/0\u001b[00m\n", + "├── \u001b[01;34m835f65ed47974d3fb3359e646b61a009\u001b[00m\n", + "├── \u001b[01;34mfbbb1fe4f9ef4b4faf370f8a946f7c60\u001b[00m\n", + "└── meta.yaml\n", + "\n", + "2 directories, 1 file\n" ] } ], "source": [ - "run_train_model_iteration(data)" + "!tree -L 1 mlruns/0" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Each of these iterations will create a new run which can be visualised through the MLFlow dashboard as per the screenshot below.\n", + "### MLmodel\n", "\n", - "![](images/mlflow-dashboard.png)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Each of these models can actually be able to found on the `mlruns` folder" + "Inside each of these folders, MLflow stores the parameters we used to train our model, any metric we logged during training, and a snapshot of our model.\n", + "If we look into one of them, we can see the following structure:" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "012c5eaa115a4f43b5e4b74cb63d5c56 meta.yaml\r\n" + "\u001b[01;34mmlruns/0/835f65ed47974d3fb3359e646b61a009\u001b[00m\n", + "├── \u001b[01;34martifacts\u001b[00m\n", + "│   └── \u001b[01;34mmodel\u001b[00m\n", + "│   ├── conda.yaml\n", + "│   ├── MLmodel\n", + "│   └── model.pkl\n", + "├── meta.yaml\n", + "├── \u001b[01;34mmetrics\u001b[00m\n", + "│   ├── mae\n", + "│   ├── r2\n", + "│   └── rmse\n", + "├── \u001b[01;34mparams\u001b[00m\n", + "│   ├── alpha\n", + "│   └── l1_ratio\n", + "└── \u001b[01;34mtags\u001b[00m\n", + " ├── mlflow.gitRepoURL\n", + " ├── mlflow.project.backend\n", + " ├── mlflow.project.entryPoint\n", + " ├── mlflow.project.env\n", + " ├── mlflow.source.git.commit\n", + " ├── mlflow.source.git.repoURL\n", + " ├── mlflow.source.name\n", + " ├── mlflow.source.type\n", + " └── mlflow.user\n", + "\n", + "5 directories, 18 files\n" ] } ], "source": [ - "!ls mlruns/0" + "!tree mlruns/0/$(ls mlruns/0 | head -1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Inside of the folders with the hash names is where we can find the artefacts of our model, which we'll be using to deploy with Seldon" + "In particular, we are interested in the `MLmodel` file stored under `artifacts/model`:" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "mlruns/0/012c5eaa115a4f43b5e4b74cb63d5c56/artifacts/\n" + "\u001b[94martifact_path\u001b[39;49;00m: model\n", + "\u001b[94mflavors\u001b[39;49;00m:\n", + " \u001b[94mpython_function\u001b[39;49;00m:\n", + " \u001b[94mdata\u001b[39;49;00m: model.pkl\n", + " \u001b[94menv\u001b[39;49;00m: conda.yaml\n", + " \u001b[94mloader_module\u001b[39;49;00m: mlflow.sklearn\n", + " \u001b[94mpython_version\u001b[39;49;00m: 3.6.9\n", + " \u001b[94msklearn\u001b[39;49;00m:\n", + " \u001b[94mpickled_model\u001b[39;49;00m: model.pkl\n", + " \u001b[94mserialization_format\u001b[39;49;00m: cloudpickle\n", + " \u001b[94msklearn_version\u001b[39;49;00m: 0.19.1\n", + "\u001b[94mrun_id\u001b[39;49;00m: 835f65ed47974d3fb3359e646b61a009\n", + "\u001b[94mutc_time_created\u001b[39;49;00m: \u001b[33m'\u001b[39;49;00m\u001b[33m2019-11-20\u001b[39;49;00m\u001b[31m \u001b[39;49;00m\u001b[33m11:15:42.706884\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\n" ] } ], "source": [ - "print(\"mlruns/0/\"+next(os.walk(\"mlruns/0\"))[1][0]+\"/artifacts/\")" + "!pygmentize -l yaml mlruns/0/$(ls mlruns/0 | head -1)/artifacts/model/MLmodel" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Now we should upload newly trained model into a public Google Bucket or S3 bucket. \n", + "This file stores the details of how the model was stored.\n", + "With this information (plus the other files in the folder), we are able to load the model back.\n", + "Seldon's MLflow server will use this information to serve this model.\n", "\n", - "We have already done this to make it simpler, which you will be able to find at `gs://seldon-models/mlflow/elasticnet_wine`" + "Now we should upload our newly trained model into a public Google Bucket or S3 bucket.\n", + "We have already done this to make it simpler, which you will be able to find at `gs://seldon-models/mlflow/model-a`." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "toc-hr-collapsed": true, + "toc-nb-collapsed": true + }, "source": [ - "## 2) Deploy your model using the Pre-packaged Moldel Server for MLFlow\n", - "Once you have a Kubernetes Cluster running with [Seldon](https://docs.seldon.io/projects/seldon-core/en/latest/workflow/install.html) and [Ambassador](https://docs.seldon.io/projects/seldon-core/en/latest/workflow/install.html#install-ambassador) running we can deploy our trained MLFlow model.\n", + "## 2. Deploy your model using the Pre-packaged Moldel Server for MLFlow\n", "\n", + "Once you have a Kubernetes Cluster running with [Seldon](https://docs.seldon.io/projects/seldon-core/en/latest/workflow/install.html) and [Ambassador](https://docs.seldon.io/projects/seldon-core/en/latest/workflow/install.html#install-ambassador) running we can deploy our trained MLFlow model.\n", "For this we have to create a Seldon definition of the model server definition, which we will break down further below.\n", "\n", "We will be using the model we updated to our google bucket (gs://seldon-models/mlflow/elasticnet_wine), but you can use your model if you uploaded it to a public bucket." @@ -383,34 +596,33 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Writing mlflow-model-server-seldon-config.yaml\n" + "\u001b[04m\u001b[36m---\u001b[39;49;00m\n", + "\u001b[94mapiVersion\u001b[39;49;00m: machinelearning.seldon.io/v1alpha2\n", + "\u001b[94mkind\u001b[39;49;00m: SeldonDeployment\n", + "\u001b[94mmetadata\u001b[39;49;00m:\n", + " \u001b[94mname\u001b[39;49;00m: mlflow-deployment\n", + "\u001b[94mspec\u001b[39;49;00m:\n", + " \u001b[94mname\u001b[39;49;00m: mlflow-deployment\n", + " \u001b[94mpredictors\u001b[39;49;00m:\n", + " - \u001b[94mgraph\u001b[39;49;00m:\n", + " \u001b[94mchildren\u001b[39;49;00m: []\n", + " \u001b[94mimplementation\u001b[39;49;00m: MLFLOW_SERVER\n", + " \u001b[94mmodelUri\u001b[39;49;00m: gs://seldon-models/mlflow/elasticnet_wine\n", + " \u001b[94mname\u001b[39;49;00m: wines-classifier\n", + " \u001b[94mname\u001b[39;49;00m: mlflow-deployment-dag\n", + " \u001b[94mreplicas\u001b[39;49;00m: 1\n" ] } ], "source": [ - "%%writefile mlflow-model-server-seldon-config.yaml\n", - "---\n", - "apiVersion: machinelearning.seldon.io/v1alpha2\n", - "kind: SeldonDeployment\n", - "metadata:\n", - " name: mlflow-deployment\n", - "spec:\n", - " name: mlflow-deployment\n", - " predictors:\n", - " - graph:\n", - " children: []\n", - " implementation: MLFLOW_SERVER\n", - " modelUri: gs://seldon-models/mlflow/elasticnet_wine\n", - " name: wines-classifier\n", - " name: mlflow-deployment-dag\n", - " replicas: 1" + "!pygmentize mlflow-model-server-seldon-config.yaml" ] }, { @@ -422,14 +634,14 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/mlflow-deployment created\r\n" + "seldondeployment.machinelearning.seldon.io/mlflow-deployment created\n" ] } ], @@ -450,19 +662,19 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "deployment \"mlflow-deployment-mlflow-deployment-dag\" successfully rolled out\r\n" + "deployment \"mlflow-deployment-mlflow-deployment-dag-77efeb1\" successfully rolled out\n" ] } ], "source": [ - "!kubectl rollout status deployment.apps/mlflow-deployment-mlflow-deployment-dag" + "!kubectl rollout status deployment.apps/mlflow-deployment-mlflow-deployment-dag-77efeb1" ] }, { @@ -474,9 +686,12 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "toc-hr-collapsed": true, + "toc-nb-collapsed": true + }, "source": [ - "## 3) Test the deployed MLFlow model by sending requests\n", + "## 3. Test the deployed MLFlow model by sending requests\n", "Now that our model is deployed in Kubernetes, we are able to send any requests." ] }, @@ -491,15 +706,15 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 32, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "ambassador LoadBalancer 10.100.227.53 localhost 80:31215/TCP,443:31622/TCP 16d\r\n", - "ambassador-admins ClusterIP 10.101.19.26 8877/TCP 16d\r\n" + "ambassador NodePort 10.97.44.51 80:30080/TCP 23h\n", + "ambassador-admin ClusterIP 10.108.207.108 8877/TCP 23h\n" ] } ], @@ -516,7 +731,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -541,7 +756,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 39, "metadata": {}, "outputs": [ { @@ -550,38 +765,26 @@ "text": [ "{\n", " \"meta\": {\n", - " \"puid\": \"4gapbfom6aa1nb6bm71jcsdo5q\",\n", + " \"puid\": \"n7i76rf930auf7u7ulhig51bu5\",\n", " \"tags\": {\n", " },\n", " \"routing\": {\n", " },\n", " \"requestPath\": {\n", - " \"wines-classifier\": \"\"\n", + " \"wines-classifier\": \"seldonio/mlflowserver_rest:0.2\"\n", " },\n", " \"metrics\": []\n", " },\n", " \"data\": {\n", " \"names\": [],\n", - " \"ndarray\": [5.655099099229193]\n", + " \"ndarray\": [5.550530190667395]\n", " }\n", "}" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " % Total % Received % Xferd Average Speed Time Time Time Current\n", - " Dload Upload Total Spent Left Speed\n", - "\r", - " 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r", - "100 354 100 250 100 104 12500 5200 --:--:-- --:--:-- --:--:-- 17700\n" - ] } ], "source": [ - "%%bash\n", - "curl -X POST -H 'Content-Type: application/json' \\\n", + "!curl -X POST -H 'Content-Type: application/json' \\\n", " -d \"{'data': {'names': [], 'ndarray': [[7.0, 0.27, 0.36, 20.7, 0.045, 45.0, 170.0, 1.001, 3.0, 0.45, 8.8]]}}\" \\\n", " http://localhost:80/seldon/default/mlflow-deployment/api/v0.1/predictions" ] @@ -595,7 +798,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 41, "metadata": {}, "outputs": [ { @@ -603,15 +806,16 @@ "output_type": "stream", "text": [ "meta {\n", - " puid: \"kt99thn77rajhquoq50jb49hmh\"\n", + " puid: \"cdjl6irq91taaavkam57g2eatu\"\n", " requestPath {\n", " key: \"wines-classifier\"\n", + " value: \"seldonio/mlflowserver_rest:0.2\"\n", " }\n", "}\n", "data {\n", " ndarray {\n", " values {\n", - " number_value: 5.655099099229193\n", + " number_value: 5.550530190667395\n", " }\n", " }\n", "}\n", @@ -632,8 +836,7 @@ "\n", "sc = SeldonClient(\n", " gateway=\"ambassador\", \n", - " gateway_endpoint=HOST + \":\" + port,\n", - " namespace=\"default\")\n", + " gateway_endpoint=HOST + \":\" + port)\n", "\n", "client_prediction = sc.predict(\n", " data=batch, \n", @@ -646,74 +849,118 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "toc-hr-collapsed": true, + "toc-nb-collapsed": true + }, "source": [ - "## 4) Deploy your second model as an A/B test\n", + "## 4. Deploy your second model as an A/B test\n", + "\n", "Now that we have a model in production, it's possible to deploy a second model as an A/B test.\n", + "Our model will also be an Elastic Net model but using a different set of parameters.\n", + "We can easily train it by leveraging MLflow:" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2019/11/20 11:38:36 INFO mlflow.projects: === Created directory /tmp/tmppr1ufom9 for downloading remote URIs passed to arguments of type 'path' ===\n", + "2019/11/20 11:38:36 INFO mlflow.projects: === Running command 'source /opt/miniconda3/bin/../etc/profile.d/conda.sh && conda activate mlflow-1ecba04797edb7e7f7212d429debd9b664c31651 1>&2 && python train.py 0.75 0.2' in run with ID '18f9f8c5d6a249f28f024011dea10e23' === \n", + "Elasticnet model (alpha=0.750000, l1_ratio=0.200000):\n", + " RMSE: 0.8118203122913661\n", + " MAE: 0.6244638140789723\n", + " R2: 0.14878415499818187\n", + "2019/11/20 11:38:37 INFO mlflow.projects: === Run (ID '18f9f8c5d6a249f28f024011dea10e23') succeeded ===\n" + ] + } + ], + "source": [ + "!mlflow run . -P alpha=0.75 -P l1_ratio=0.2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we did before, we will now need to upload our model to a cloud bucket.\n", + "To speed things up, we already have done so and the second model is now accessible in `gs://seldon-models/mlflow/model-b`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### A/B test\n", "\n", - "By leveraging this, we will be redirecting 20% of the traffic to the new model.\n", + "We will deploy our second model as an A/B test.\n", + "In particular, we will redirect 20% of the traffic to the new model.\n", "\n", - "This can be done by simply adding a `traffic` attribute as shown below. " + "This can be done by simply adding a `traffic` attribute on our `SeldonDeployment` spec:" ] }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 44, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Overwriting ab-test-mlflow-model-server-seldon-config.yaml\n" + "\u001b[04m\u001b[36m---\u001b[39;49;00m\n", + "\u001b[94mapiVersion\u001b[39;49;00m: machinelearning.seldon.io/v1alpha2\n", + "\u001b[94mkind\u001b[39;49;00m: SeldonDeployment\n", + "\u001b[94mmetadata\u001b[39;49;00m:\n", + " \u001b[94mname\u001b[39;49;00m: mlflow-deployment\n", + "\u001b[94mspec\u001b[39;49;00m:\n", + " \u001b[94mname\u001b[39;49;00m: mlflow-deployment\n", + " \u001b[94mpredictors\u001b[39;49;00m:\n", + " - \u001b[94mgraph\u001b[39;49;00m:\n", + " \u001b[94mchildren\u001b[39;49;00m: []\n", + " \u001b[94mimplementation\u001b[39;49;00m: MLFLOW_SERVER\n", + " \u001b[94mmodelUri\u001b[39;49;00m: gs://seldon-models/mlflow/model-a\n", + " \u001b[94mname\u001b[39;49;00m: wines-classifier\n", + " \u001b[94mname\u001b[39;49;00m: a-mlflow-deployment-dag\n", + " \u001b[94mreplicas\u001b[39;49;00m: 1\n", + " \u001b[94mtraffic\u001b[39;49;00m: 80\n", + " - \u001b[94mgraph\u001b[39;49;00m:\n", + " \u001b[94mchildren\u001b[39;49;00m: []\n", + " \u001b[94mimplementation\u001b[39;49;00m: MLFLOW_SERVER\n", + " \u001b[94mmodelUri\u001b[39;49;00m: gs://seldon-models/mlflow/model-b\n", + " \u001b[94mname\u001b[39;49;00m: wines-classifier\n", + " \u001b[94mname\u001b[39;49;00m: b-mlflow-deployment-dag\n", + " \u001b[94mreplicas\u001b[39;49;00m: 1\n", + " \u001b[94mtraffic\u001b[39;49;00m: 20\n" ] } ], "source": [ - "%%writefile ab-test-mlflow-model-server-seldon-config.yaml\n", - "---\n", - "apiVersion: machinelearning.seldon.io/v1alpha2\n", - "kind: SeldonDeployment\n", - "metadata:\n", - " name: mlflow-deployment\n", - "spec:\n", - " name: mlflow-deployment\n", - " predictors:\n", - " - graph:\n", - " children: []\n", - " implementation: MLFLOW_SERVER\n", - " modelUri: gs://seldon-models/mlflow/elasticnet_wine\n", - " name: wines-classifier\n", - " name: a-mlflow-deployment-dag\n", - " replicas: 1\n", - " traffic: 20\n", - " - graph:\n", - " children: []\n", - " implementation: MLFLOW_SERVER\n", - " modelUri: gs://seldon-models/mlflow/elasticnet_wine\n", - " name: wines-classifier\n", - " name: b-mlflow-deployment-dag\n", - " replicas: 1\n", - " traffic: 80" + "!pygmentize ab-test-mlflow-model-server-seldon-config.yaml" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "And similar to the model above, we only need to run the command to run it" + "And similar to the model above, we only need to run the following to deploy it:" ] }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 47, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/mlflow-deployment configured\r\n" + "seldondeployment.machinelearning.seldon.io/mlflow-deployment created\n" ] } ], @@ -732,20 +979,17 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 51, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "NAME READY STATUS RESTARTS AGE\r\n", - "ambassador-6657ccd4f6-4z6xh 1/1 Running 11 16d\r\n", - "ambassador-6657ccd4f6-5mc46 1/1 Running 11 16d\r\n", - "ambassador-6657ccd4f6-qqfgn 1/1 Running 13 16d\r\n", - "mlflow-deployment-a-mlflow-deployment-dag-68d4c9fcf5-crd9q 2/2 Running 0 2m25s\r\n", - "mlflow-deployment-b-mlflow-deployment-dag-8cdcccbfc-rhc4p 2/2 Running 0 2m25s\r\n", - "seldon-operator-controller-manager-0 1/1 Running 0 110m\r\n" + "NAME READY STATUS RESTARTS AGE\n", + "ambassador-5d97b7df6f-tkrhq 1/1 Running 0 24h\n", + "mlflow-deployment-a-mlflow-deployment-dag-77efeb1-56dd56dcpx54t 0/2 Init:0/1 0 6s\n", + "mlflow-deployment-b-mlflow-deployment-dag-77efeb1-86cb459drl7fw 0/2 Init:0/1 0 6s\n" ] } ], @@ -754,11 +998,11 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "## 5) Visualise and monitor the performance of your models using Seldon Analytics\n", + "## 5. Visualise and monitor the performance of your models using Seldon Analytics\n", + "\n", "This section is optional, but by following the instructions you will be able to visualise the performance of both models as per the chart below.\n", "\n", "In order for this example to work you need to install and run the [Grafana Analytics package for Seldon Core](https://docs.seldon.io/projects/seldon-core/en/latest/analytics/analytics.html#helm-analytics-chart).\n", @@ -788,41 +1032,89 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "Now that we can access grafana, you have to go to the prediction analytics dashboard, where you'll be able to see metrics.\n", + "Now that we have both models running in our Kubernetes cluster, we can analyse their performance using Seldon Core's integration with Prometheus and Grafana.\n", + "To do so, we will iterate over the training set (which can be found in `wine-quality.csv`), making a request and sending the feedback of the prediction.\n", "\n", - "Now we can run the following `while True` loop to start sending some data:" + "Since the `/feedback` endpoint requires a `reward` signal (i.e. the higher the better), we will simulate one as:\n", + "\n", + "$$\n", + " R(x_{n})\n", + " = \\begin{cases}\n", + " \\frac{1}{(y_{n} - f(x_{n}))^{2}} &, y_{n} \\neq f(x_{n}) \\\\\n", + " 500 &, y_{n} = f(x_{n})\n", + " \\end{cases}\n", + "$$\n", + "\n", + ", where $R(x_{n})$ is the reward for input point $x_{n}$, $f(x_{n})$ is our trained model and $y_{n}$ is the actual value." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "0 [4.949928760465064]\n", + "1 [2.33866485520918]\n", + "2 [16.671295276036165]\n", + "3 [11.360528710955778]\n", + "4 [10.762015969288063]\n", + " ... \n", + "4893 [270.7374890482452]\n", + "4894 [1.8348875422756648]\n", + "4895 [3.872377496349884]\n", + "4896 [1.9544204216470193]\n", + "4897 [22.25374886390087]\n", + "Length: 4898, dtype: object" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "while True:\n", - " client_prediction = sc.predict(\n", - " data=batch, \n", + "def _get_reward(y, y_pred):\n", + " if y == y_pred:\n", + " return 500 \n", + " \n", + " return 1 / np.square(y - y_pred)\n", + "\n", + "def _test_row(row):\n", + " input_features = row[:-1]\n", + " feature_names = input_features.index.to_list()\n", + " X = input_features.values.reshape(1, -1)\n", + " y = row[-1].reshape(1, -1)\n", + " \n", + " # Note that we are re-using the SeldonClient defined previously\n", + " r = sc.predict(\n", + " deployment_name=\"mlflow-deployment\",\n", + " data=X,\n", + " names=feature_names)\n", + " \n", + " y_pred = r.response.data.tensor.values\n", + " reward = _get_reward(y, y_pred)\n", + " sc.feedback(\n", " deployment_name=\"mlflow-deployment\",\n", - " names=[],\n", - " payload_type=payload_type)\n", + " prediction_request=r.request,\n", + " prediction_response=r.response,\n", + " reward=reward)\n", + " \n", + " return reward[0]\n", "\n", - "print(client_prediction.response)" + "data.apply(_test_row, axis=1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "You should now be able to see the metrics reflected as per the chart below.\n", - "\n", - "In the chart you can visualise on the bottom lef the requests per second, which shows the different traffic breakdown we specified.\n", - "\n", - "You are able to add your own custom metrics, and try out other more complex deployments by following further guides at https://docs.seldon.io/projects/seldon-core/en/latest/workflow/README.html" + "You should now be able to see Seldon's pre-built Grafana dashboard." ] }, { @@ -831,13 +1123,33 @@ "source": [ "![](images/grafana-mlflow.jpg)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In bottom of the dashboard you can see the following charts: \n", + "\n", + "- On the left: the requests per second, which shows the different traffic breakdown we specified.\n", + "- On the center: the reward, where you can see how model `a` outperforms model `b` by a large margin.\n", + "- On the right, the latency for each one of them.\n", + "\n", + "You are able to add your own custom metrics, and try out other more complex deployments by following further guides at https://docs.seldon.io/projects/seldon-core/en/latest/workflow/README.html" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "mirror1", "language": "python", - "name": "python3" + "name": "mirror1" }, "language_info": { "codemirror_mode": { @@ -849,7 +1161,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.6.9" }, "varInspector": { "cols": { @@ -882,5 +1194,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/examples/models/mlflow_server_ab_test_ambassador/train.py b/examples/models/mlflow_server_ab_test_ambassador/train.py new file mode 100644 index 0000000000..1ea6807721 --- /dev/null +++ b/examples/models/mlflow_server_ab_test_ambassador/train.py @@ -0,0 +1,67 @@ +# The data set used in this example is from http://archive.ics.uci.edu/ml/datasets/Wine+Quality +# P. Cortez, A. Cerdeira, F. Almeida, T. Matos and J. Reis. +# Modeling wine preferences by data mining from physicochemical properties. In Decision Support Systems, Elsevier, 47(4):547-553, 2009. + +import os +import warnings +import sys + +import pandas as pd +import numpy as np +from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score +from sklearn.model_selection import train_test_split +from sklearn.linear_model import ElasticNet + +import mlflow +import mlflow.sklearn + + +def eval_metrics(actual, pred): + rmse = np.sqrt(mean_squared_error(actual, pred)) + mae = mean_absolute_error(actual, pred) + r2 = r2_score(actual, pred) + return rmse, mae, r2 + + +if __name__ == "__main__": + warnings.filterwarnings("ignore") + np.random.seed(40) + + # Read the wine-quality csv file (make sure you're running this from the root of MLflow!) + wine_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "wine-quality.csv" + ) + data = pd.read_csv(wine_path) + + # Split the data into training and test sets. (0.75, 0.25) split. + train, test = train_test_split(data) + + # The predicted column is "quality" which is a scalar from [3, 9] + train_x = train.drop(["quality"], axis=1) + test_x = test.drop(["quality"], axis=1) + train_y = train[["quality"]] + test_y = test[["quality"]] + + alpha = float(sys.argv[1]) if len(sys.argv) > 1 else 0.5 + l1_ratio = float(sys.argv[2]) if len(sys.argv) > 2 else 0.5 + + with mlflow.start_run(): + lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42) + lr.fit(train_x, train_y) + + predicted_qualities = lr.predict(test_x) + + (rmse, mae, r2) = eval_metrics(test_y, predicted_qualities) + + print("Elasticnet model (alpha=%f, l1_ratio=%f):" % (alpha, l1_ratio)) + print(" RMSE: %s" % rmse) + print(" MAE: %s" % mae) + print(" R2: %s" % r2) + + mlflow.log_param("alpha", alpha) + mlflow.log_param("l1_ratio", l1_ratio) + mlflow.log_metric("rmse", rmse) + mlflow.log_metric("r2", r2) + mlflow.log_metric("mae", mae) + + mlflow.sklearn.log_model(lr, "model")