Update e2e tests and add Combiner to python wrappers

examples/combiners/mnist_combiner/MnistCombiner.py

@@ -0,0 +1,12 @@
+import logging
+logger = logging.getLogger(__name__)
+
+class MnistCombiner(object):
+    def __init__(self, metrics_ok=True):
+        print("MNIST Combiner Init called")
+
+    def aggregate(self, Xs, features_names):
+        print("MNIST Combiner aggregate called")
+        logger.info(Xs)
+        return (Xs[0]+Xs[1])/2.0
+

examples/combiners/mnist_combiner/environment_grpc

@@ -0,0 +1,4 @@
+MODEL_NAME=MnistCombiner
+API_TYPE=GRPC
+SERVICE_TYPE=COMBINER
+PERSISTENCE=0

examples/combiners/mnist_combiner/environment_rest

@@ -0,0 +1,4 @@
+MODEL_NAME=MnistCombiner
+API_TYPE=REST
+SERVICE_TYPE=COMBINER
+PERSISTENCE=0

examples/combiners/mnist_combiner/mnist_combiner.ipynb

@@ -0,0 +1,331 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# MNIST Combiner\n",
+    "\n",
+    "Combines two models, an SKLearn model and a Tensorflow model for MNIST. The combination does a simple average of the two models."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mlogging\u001b[39;49;00m\r\n",
+      "logger = logging.getLogger(\u001b[31m__name__\u001b[39;49;00m)\r\n",
+      "\r\n",
+      "\u001b[34mclass\u001b[39;49;00m \u001b[04m\u001b[32mMnistCombiner\u001b[39;49;00m(\u001b[36mobject\u001b[39;49;00m):\r\n",
+      "    \u001b[34mdef\u001b[39;49;00m \u001b[32m__init__\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m, metrics_ok=\u001b[36mTrue\u001b[39;49;00m):\r\n",
+      "        \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33mMNIST Combiner Init called\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\r\n",
+      "\r\n",
+      "    \u001b[34mdef\u001b[39;49;00m \u001b[32maggregate\u001b[39;49;00m(\u001b[36mself\u001b[39;49;00m, Xs, features_names):\r\n",
+      "        \u001b[34mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33mMNIST Combiner aggregate called\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\r\n",
+      "        logger.info(Xs)\r\n",
+      "        \u001b[34mreturn\u001b[39;49;00m (Xs[\u001b[34m0\u001b[39;49;00m]+Xs[\u001b[34m1\u001b[39;49;00m])/\u001b[34m2.0\u001b[39;49;00m\r\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pygmentize MnistCombiner.py"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Start Minikube"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!minikube start --memory 4096 --feature-gates=CustomResourceValidation=true --extra-config=apiserver.Authorization.Mode=RBAC"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kubectl config set-context $(kubectl config current-context) --namespace=seldon"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Build Combiner image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---> Installing application source...\n",
+      "Build completed successfully\n"
+     ]
+    }
+   ],
+   "source": [
+    "!eval $(minikube docker-env) && s2i build -E environment_rest . seldonio/seldon-core-s2i-python36:0.4-SNAPSHOT seldonio/mnistcombiner_rest:0.1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Install Helm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kubectl -n kube-system create sa tiller\n",
+    "!kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller\n",
+    "!helm init --service-account tiller"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kubectl rollout status deploy/tiller-deploy -n kube-system"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Install Seldon Core"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kubectl create namespace seldon"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!helm install ../../../helm-charts/seldon-core-crd --name seldon-core-crd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!helm install ../../../helm-charts/seldon-core --name seldon-core --namespace seldon \\\n",
+    "    --set ambassador.enabled=true"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!kubectl rollout status deploy/seldon-core-seldon-cluster-manager\n",
+    "!kubectl rollout status deploy/seldon-core-seldon-apiserver\n",
+    "!kubectl rollout status deploy/seldon-core-ambassador"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To send requests to Ambassador ingress in another terminal run:\n",
+    "    \n",
+    "```\n",
+    "kubectl port-forward $(kubectl get pods -n seldon -l service=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8002:8080\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "WARNING:tensorflow:From /home/clive/work/seldon-core/fork-seldon-core/examples/combiners/mnist_combiner/utils.py:57: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
+      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "Please write your own downloading logic.\n",
+      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "Please use tf.data to implement this functionality.\n",
+      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
+      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "Please use tf.data to implement this functionality.\n",
+      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
+      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "Please use tf.one_hot on tensors.\n",
+      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
+      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
+      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
+      "Instructions for updating:\n",
+      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
+     ]
+    }
+   ],
+   "source": [
+    "%matplotlib inline\n",
+    "import utils\n",
+    "from visualizer import get_graph\n",
+    "mnist = utils.download_mnist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "get_graph(\"mnist_combiner.json\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pygmentize mnist_combiner.json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "seldondeployment.machinelearning.seldon.io/mnistcombo created\r\n"
+     ]
+    }
+   ],
+   "source": [
+    "!kubectl apply -f mnist_combiner.json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "deployment \"mnistcombo-mnistcombo-3715bc4\" successfully rolled out\r\n"
+     ]
+    }
+   ],
+   "source": [
+    "!kubectl rollout status deploy/mnistcombo-mnistcombo-3715bc4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADmNJREFUeJzt3W+MVGWWx/HfsYUQgShI4yLgNhpYQQiMVsgmrhuNYRRD/PMCM0QnjE7seQEJJr7wT9QxJmsI7jBO4jIJs4MwCeNAMsNqjKwYskGJk5HSAIoIY0yv9EKgDZqh9cVIc/ZFX0yLXU8VVbfqVnO+n4RU1T33qXss+XGr6qmqx9xdAOK5qOgGABSD8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCOriVh5s0qRJ3tXV1cpDAqH09PTo888/t1r2bSj8Zna7pF9J6pD0n+6+OrV/V1eXyuVyI4cEkFAqlWret+6n/WbWIek/JC2WNEfSMjObU+/9AWitRl7zL5T0ibt/6u5/l/QHSXfl0xaAZmsk/FMlHRlyuzfb9h1m1m1mZTMr9/X1NXA4AHlqJPzDvanwve8Hu/t6dy+5e6mzs7OBwwHIUyPh75U0fcjtaZKONtYOgFZpJPx7JM00sxlmNlrSjyS9mk9bAJqt7qk+dz9tZislvaHBqb4N7n4gt84ANFVD8/zu/rqk13PqBUAL8fFeICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Jq6RLdqE9/f3+y/vXXX1esTZ48Oe92zov79xZx+tbp06eTY/fu3Zusr1u3LlnfuHFjxdqpU6eSY8eNG5esXwg48wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUA3N85tZj6RTkgYknXb3Uh5N4bsefPDBZH379u0VaytXrkyOve666+rq6aw33ngjWT9y5EjF2q5duxo69ujRo5P1VatWVayNGTOmoWNfCPL4kM8t7v55DvcDoIV42g8E1Wj4XdIOM3vPzLrzaAhAazT6tP9Gdz9qZpMlvWlmH7v7W0N3yP5R6Jakq666qsHDAchLQ2d+dz+aXZ6QtE3SwmH2We/uJXcvdXZ2NnI4ADmqO/xmNtbMxp+9LumHkj7MqzEAzdXI0/4rJG0zs7P383t3/+9cugLQdHWH390/lTQ/x15QQbX57NT3/VevXp13O+clNZ/e1dWVHHvLLbck648//niyPnPmzGQ9Oqb6gKAIPxAU4QeCIvxAUIQfCIrwA0Hx090jwA033JCsb968uWLtnnvuSY6dPz89W3vppZcm67Nnz07WZ82aVbE2Y8aM5Fg0F2d+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiKef4R4O2336577JNPPpmsX3/99XXfN0Y2zvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBTz/G1gYGAgWe/t7U3WR40aVVcNsXHmB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgqs7zm9kGSUsknXD3udm2iZK2SOqS1CPpXnf/onltXth2796drO/ZsydZv/rqqyvW5s2bV1dPuPDVcubfKOn2c7Y9Jmmnu8+UtDO7DWAEqRp+d39L0slzNt8laVN2fZOku3PuC0CT1fua/wp3PyZJ2eXk/FoC0ApNf8PPzLrNrGxm5b6+vmYfDkCN6g3/cTObIknZ5YlKO7r7encvuXups7OzzsMByFu94X9V0vLs+nJJr+TTDoBWqRp+M3tZ0p8l/ZOZ9ZrZTyWtlrTIzP4qaVF2G8AIUnWe392XVSjdmnMvYe3fv7+h8bfeOjL/V3z22WfJ+pVXXpmsX3wxP0fRCD7hBwRF+IGgCD8QFOEHgiL8QFCEHwiKuZI2sHPnzobGHz58uGLtueeea+i+jx49mqyfPHnud76+69ChQxVrBw4cSI5dsWJFsr5mzZpkvaOjI1mPjjM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwTFPH8LfPXVV8n6vn37Grr/Xbt21VVrhVmzZlWsjR8/Pjl27dq1yfrUqVOT9VWrVlWs8RkAzvxAWIQfCIrwA0ERfiAowg8ERfiBoAg/EBTz/C1Q7TvxPT09TTt2ap5dku68885kffr06cn6okWLkvVrrrmmYi31OwSSdMcddyTrjzzySLI+bty4irXu7u7k2Ag48wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUFXn+c1sg6Qlkk64+9xs2zOSHpLUl+32hLu/3qwmR7pyudzQ+JtuuilZf/755yvW5s+fnxw7ZsyYunrKw9y5c5P1Bx54IFl/9tlnk/XU5wDuu+++5NixY8cm6xeCWs78GyXdPsz2X7r7guwPwQdGmKrhd/e3JKWXZQEw4jTymn+lme03sw1mNiG3jgC0RL3h/7WkayQtkHRM0i8q7Whm3WZWNrNyX19fpd0AtFhd4Xf34+4+4O5nJP1G0sLEvuvdveTupc7Oznr7BJCzusJvZlOG3LxH0of5tAOgVWqZ6ntZ0s2SJplZr6SfS7rZzBZIckk9kn7WxB4BNIG5e8sOViqVvNE575FoYGAgWX/nnXeS9Tlz5iTrl19++Xn3NBIcOXIkWa/2uPT391esbd26NTl26dKlyXq7KpVKKpfLVsu+fMIPCIrwA0ERfiAowg8ERfiBoAg/EBQ/3d0C1ZaDrvaV3aiq/Wz4tGnTkvWPP/64Yq3asukRcOYHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaCY58eIddlllxXdwojGmR8IivADQRF+ICjCDwRF+IGgCD8QFOEHgmKeH23rxRdfTNbffffduu978uTJdY+9UHDmB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgqs7zm9l0Sb+T9A+Szkha7+6/MrOJkrZI6pLUI+led/+iea0Wa/fu3RVrDz30UHLshAkTkvXXXnstWZ84cWKyPlJt2bIlWX/qqaeS9TNnziTrjz76aMXa4sWLk2MjqOXMf1rSI+4+W9I/S1phZnMkPSZpp7vPlLQzuw1ghKgafnc/5u7vZ9dPSTooaaqkuyRtynbbJOnuZjUJIH/n9ZrfzLok/UDSXyRd4e7HpMF/ICTxeUlgBKk5/GY2TtIfJT3s7n87j3HdZlY2s3JfX189PQJogprCb2ajNBj8ze7+p2zzcTObktWnSDox3Fh3X+/uJXcvdXZ25tEzgBxUDb+ZmaTfSjro7muHlF6VtDy7vlzSK/m3B6BZavlK742SfizpAzPbm217QtJqSVvN7KeSPpO0tDkttof+/v6KtdRS0LWoNu300ksvJetz5sxp6PiNqPbfvm3btoq1NWvWJMd++eWXyfry5cuT9aeffrpibfCcFlvV8Lv7bkmVHqlb820HQKvwCT8gKMIPBEX4gaAIPxAU4QeCIvxAUPx0d41uu+22irWPPvooOfbhhx9O1nfs2JGsL1iwIFnv6OhI1ptpYGAgWf/mm2/qvu9q8/jr1q1L1i+55JK6jx0BZ34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIp5/hqlvv89e/bs5Njt27cn64cPH07W9+3bl6wfOnSoYu2FF15Ijv3ii8Z+bX3evHnJ+rXXXluxdv/99yfHLlmyJFm/6CLOXY3g0QOCIvxAUIQfCIrwA0ERfiAowg8ERfiBoJjnb4Fq89GpufBa6imp365HbJz5gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiCoquE3s+lm9j9mdtDMDpjZqmz7M2b2f2a2N/tzR/PbBZCXWj7kc1rSI+7+vpmNl/Semb2Z1X7p7v/evPYANEvV8Lv7MUnHsuunzOygpKnNbgxAc53Xa34z65L0A0l/yTatNLP9ZrbBzCZUGNNtZmUzK/f19TXULID81Bx+Mxsn6Y+SHnb3v0n6taRrJC3Q4DODXww3zt3Xu3vJ3UudnZ05tAwgDzWF38xGaTD4m939T5Lk7sfdfcDdz0j6jaSFzWsTQN5qebffJP1W0kF3Xztk+5Qhu90j6cP82wPQLLW823+jpB9L+sDM9mbbnpC0zMwWSHJJPZJ+1pQOATRFLe/275Y03I/Wv55/OwBahU/4AUERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgjJ3b93BzPok/e+QTZMkfd6yBs5Pu/bWrn1J9FavPHv7R3ev6ffyWhr+7x3crOzupcIaSGjX3tq1L4ne6lVUbzztB4Ii/EBQRYd/fcHHT2nX3tq1L4ne6lVIb4W+5gdQnKLP/AAKUkj4zex2MztkZp+Y2WNF9FCJmfWY2QfZysPlgnvZYGYnzOzDIdsmmtmbZvbX7HLYZdIK6q0tVm5OrCxd6GPXbitet/xpv5l1SDosaZGkXkl7JC1z949a2kgFZtYjqeTuhc8Jm9m/SuqX9Dt3n5ttWyPppLuvzv7hnODuj7ZJb89I6i965eZsQZkpQ1eWlnS3pJ+owMcu0de9KuBxK+LMv1DSJ+7+qbv/XdIfJN1VQB9tz93fknTynM13SdqUXd+kwb88LVeht7bg7sfc/f3s+ilJZ1eWLvSxS/RViCLCP1XSkSG3e9VeS367pB1m9p6ZdRfdzDCuyJZNP7t8+uSC+zlX1ZWbW+mclaXb5rGrZ8XrvBUR/uFW/2mnKYcb3f16SYslrcie3qI2Na3c3CrDrCzdFupd8TpvRYS/V9L0IbenSTpaQB/Dcvej2eUJSdvUfqsPHz+7SGp2eaLgfr7VTis3D7eytNrgsWunFa+LCP8eSTPNbIaZjZb0I0mvFtDH95jZ2OyNGJnZWEk/VPutPvyqpOXZ9eWSXimwl+9ol5WbK60srYIfu3Zb8bqQD/lkUxkvSOqQtMHd/63lTQzDzK7W4NleGlzE9PdF9mZmL0u6WYPf+jou6eeS/kvSVklXSfpM0lJ3b/kbbxV6u1mDT12/Xbn57GvsFvf2L5LelvSBpDPZ5ic0+Pq6sMcu0dcyFfC48Qk/ICg+4QcERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+IKj/B237/BEUH07gAAAAAElFTkSuQmCC\n",
+      "text/plain": [
+       "<matplotlib.figure.Figure at 0x7efe76439c88>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'meta': {'puid': 'h4tuqh36rdmu3bcdlcpc0r3mke', 'tags': {}, 'routing': {'combiner': -1}, 'requestPath': {'tfmodel': 'seldonio/deep-mnist:0.1', 'skmodel': 'seldonio/sk-mnist:0.1', 'combiner': 'seldonio/mnistcombiner_rest:0.1'}, 'metrics': []}, 'data': {'names': ['class:0', 'class:1', 'class:2', 'class:3', 'class:4', 'class:5', 'class:6', 'class:7', 'class:8', 'class:9'], 'ndarray': [[0.01777702378264318, 2.8567853860295145e-06, 0.017218996949183444, 0.0812305503214399, 0.0010864619398489594, 0.8603562653064728, 0.00041601501288823783, 2.683608215647837e-07, 0.02169143408536911, 0.00022013898706063628]]}}\n"
+     ]
+    }
+   ],
+   "source": [
+    "utils.predict_rest_mnist(mnist,\"mnistcombo\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}