sdk/python/examples/kubeflow-tfjob-sdk.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sample for Kubeflow TFJob SDK"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a sample for Kubeflow TFJob SDK `kubeflow-tfjob`.\n",
    "\n",
    "The notebook shows how to use Kubeflow TFJob SDK to create, get, wait, check and delete tfjob."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kubernetes.client import V1PodTemplateSpec\n",
    "from kubernetes.client import V1ObjectMeta\n",
    "from kubernetes.client import V1PodSpec\n",
    "from kubernetes.client import V1Container\n",
    "\n",
    "from kubeflow.training import constants\n",
    "from kubeflow.training.utils import utils\n",
    "from kubeflow.training import V1ReplicaSpec\n",
    "from kubeflow.training import V1TFJob\n",
    "from kubeflow.training import V1TFJobSpec\n",
    "from kubeflow.training import V1RunPolicy\n",
    "from kubeflow.training import TFJobClient"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define namespace where tfjob needs to be created to. If not specified, below function defines namespace to the current one where SDK is running in the cluster, otherwise it will deploy to default namespace."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "namespace = utils.get_default_target_namespace()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define TFJob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The demo only creates a worker of TFJob to run mnist sample."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "container = V1Container(\n",
    "    name=\"tensorflow\",\n",
    "    image=\"gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0\",\n",
    "    command=[\n",
    "        \"python\",\n",
    "        \"/var/tf_mnist/mnist_with_summaries.py\",\n",
    "        \"--log_dir=/train/logs\", \"--learning_rate=0.01\",\n",
    "        \"--batch_size=150\"\n",
    "        ]\n",
    ")\n",
    "\n",
    "worker = V1ReplicaSpec(\n",
    "    replicas=2,\n",
    "    restart_policy=\"Never\",\n",
    "    template=V1PodTemplateSpec(\n",
    "        spec=V1PodSpec(\n",
    "            containers=[container]\n",
    "        )\n",
    "    )\n",
    ")\n",
    "\n",
    "chief = V1ReplicaSpec(\n",
    "    replicas=1,\n",
    "    restart_policy=\"Never\",\n",
    "    template=V1PodTemplateSpec(\n",
    "        spec=V1PodSpec(\n",
    "            containers=[container]\n",
    "        )\n",
    "    )\n",
    ")\n",
    "\n",
    "ps = V1ReplicaSpec(\n",
    "    replicas=1,\n",
    "    restart_policy=\"Never\",\n",
    "    template=V1PodTemplateSpec(\n",
    "        spec=V1PodSpec(\n",
    "            containers=[container]\n",
    "        )\n",
    "    )\n",
    ")\n",
    "\n",
    "tfjob = V1TFJob(\n",
    "    api_version=\"kubeflow.org/v1\",\n",
    "    kind=\"TFJob\",\n",
    "    metadata=V1ObjectMeta(name=\"mnist\",namespace=namespace),\n",
    "    spec=V1TFJobSpec(\n",
    "        run_policy=V1RunPolicy(clean_pod_policy=\"None\"),\n",
    "        tf_replica_specs={\"Worker\": worker,\n",
    "                          \"Chief\": chief,\n",
    "                          \"PS\": ps}\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create TFJob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'apiVersion': 'kubeflow.org/v1',\n",
       " 'kind': 'TFJob',\n",
       " 'metadata': {'creationTimestamp': '2021-10-02T19:02:08Z',\n",
       "  'generation': 1,\n",
       "  'managedFields': [{'apiVersion': 'kubeflow.org/v1',\n",
       "    'fieldsType': 'FieldsV1',\n",
       "    'fieldsV1': {'f:spec': {'.': {},\n",
       "      'f:runPolicy': {'.': {}, 'f:cleanPodPolicy': {}},\n",
       "      'f:tfReplicaSpecs': {'.': {},\n",
       "       'f:Chief': {'.': {},\n",
       "        'f:replicas': {},\n",
       "        'f:restartPolicy': {},\n",
       "        'f:template': {'.': {}, 'f:spec': {'.': {}, 'f:containers': {}}}},\n",
       "       'f:PS': {'.': {},\n",
       "        'f:replicas': {},\n",
       "        'f:restartPolicy': {},\n",
       "        'f:template': {'.': {}, 'f:spec': {'.': {}, 'f:containers': {}}}},\n",
       "       'f:Worker': {'.': {},\n",
       "        'f:replicas': {},\n",
       "        'f:restartPolicy': {},\n",
       "        'f:template': {'.': {}, 'f:spec': {'.': {}, 'f:containers': {}}}}}}},\n",
       "    'manager': 'OpenAPI-Generator',\n",
       "    'operation': 'Update',\n",
       "    'time': '2021-10-02T19:02:08Z'}],\n",
       "  'name': 'mnist',\n",
       "  'namespace': 'default',\n",
       "  'resourceVersion': '6042',\n",
       "  'uid': '4a0b9764-b5c4-4d30-95c3-d3c56d342803'},\n",
       " 'spec': {'runPolicy': {'cleanPodPolicy': 'None'},\n",
       "  'tfReplicaSpecs': {'Chief': {'replicas': 1,\n",
       "    'restartPolicy': 'Never',\n",
       "    'template': {'spec': {'containers': [{'command': ['python',\n",
       "         '/var/tf_mnist/mnist_with_summaries.py',\n",
       "         '--log_dir=/train/logs',\n",
       "         '--learning_rate=0.01',\n",
       "         '--batch_size=150'],\n",
       "        'image': 'gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0',\n",
       "        'name': 'tensorflow'}]}}},\n",
       "   'PS': {'replicas': 1,\n",
       "    'restartPolicy': 'Never',\n",
       "    'template': {'spec': {'containers': [{'command': ['python',\n",
       "         '/var/tf_mnist/mnist_with_summaries.py',\n",
       "         '--log_dir=/train/logs',\n",
       "         '--learning_rate=0.01',\n",
       "         '--batch_size=150'],\n",
       "        'image': 'gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0',\n",
       "        'name': 'tensorflow'}]}}},\n",
       "   'Worker': {'replicas': 2,\n",
       "    'restartPolicy': 'Never',\n",
       "    'template': {'spec': {'containers': [{'command': ['python',\n",
       "         '/var/tf_mnist/mnist_with_summaries.py',\n",
       "         '--log_dir=/train/logs',\n",
       "         '--learning_rate=0.01',\n",
       "         '--batch_size=150'],\n",
       "        'image': 'gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0',\n",
       "        'name': 'tensorflow'}]}}}}}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfjob_client = TFJobClient()\n",
    "tfjob_client.create(tfjob, namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the created TFJob "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'apiVersion': 'kubeflow.org/v1',\n",
       " 'kind': 'TFJob',\n",
       " 'metadata': {'creationTimestamp': '2021-10-02T19:02:08Z',\n",
       "  'generation': 1,\n",
       "  'managedFields': [{'apiVersion': 'kubeflow.org/v1',\n",
       "    'fieldsType': 'FieldsV1',\n",
       "    'fieldsV1': {'f:spec': {'.': {},\n",
       "      'f:runPolicy': {'.': {}, 'f:cleanPodPolicy': {}},\n",
       "      'f:tfReplicaSpecs': {'.': {},\n",
       "       'f:Chief': {'.': {},\n",
       "        'f:replicas': {},\n",
       "        'f:restartPolicy': {},\n",
       "        'f:template': {'.': {}, 'f:spec': {'.': {}, 'f:containers': {}}}},\n",
       "       'f:PS': {'.': {},\n",
       "        'f:replicas': {},\n",
       "        'f:restartPolicy': {},\n",
       "        'f:template': {'.': {}, 'f:spec': {'.': {}, 'f:containers': {}}}},\n",
       "       'f:Worker': {'.': {},\n",
       "        'f:replicas': {},\n",
       "        'f:restartPolicy': {},\n",
       "        'f:template': {'.': {}, 'f:spec': {'.': {}, 'f:containers': {}}}}}}},\n",
       "    'manager': 'OpenAPI-Generator',\n",
       "    'operation': 'Update',\n",
       "    'time': '2021-10-02T19:02:08Z'},\n",
       "   {'apiVersion': 'kubeflow.org/v1',\n",
       "    'fieldsType': 'FieldsV1',\n",
       "    'fieldsV1': {'f:status': {'.': {},\n",
       "      'f:conditions': {},\n",
       "      'f:replicaStatuses': {'.': {},\n",
       "       'f:Chief': {'.': {}, 'f:active': {}},\n",
       "       'f:PS': {'.': {}, 'f:active': {}},\n",
       "       'f:Worker': {}},\n",
       "      'f:startTime': {}}},\n",
       "    'manager': 'manager',\n",
       "    'operation': 'Update',\n",
       "    'time': '2021-10-02T19:02:10Z'}],\n",
       "  'name': 'mnist',\n",
       "  'namespace': 'default',\n",
       "  'resourceVersion': '6105',\n",
       "  'uid': '4a0b9764-b5c4-4d30-95c3-d3c56d342803'},\n",
       " 'spec': {'runPolicy': {'cleanPodPolicy': 'None'},\n",
       "  'tfReplicaSpecs': {'Chief': {'replicas': 1,\n",
       "    'restartPolicy': 'Never',\n",
       "    'template': {'spec': {'containers': [{'command': ['python',\n",
       "         '/var/tf_mnist/mnist_with_summaries.py',\n",
       "         '--log_dir=/train/logs',\n",
       "         '--learning_rate=0.01',\n",
       "         '--batch_size=150'],\n",
       "        'image': 'gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0',\n",
       "        'name': 'tensorflow'}]}}},\n",
       "   'PS': {'replicas': 1,\n",
       "    'restartPolicy': 'Never',\n",
       "    'template': {'spec': {'containers': [{'command': ['python',\n",
       "         '/var/tf_mnist/mnist_with_summaries.py',\n",
       "         '--log_dir=/train/logs',\n",
       "         '--learning_rate=0.01',\n",
       "         '--batch_size=150'],\n",
       "        'image': 'gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0',\n",
       "        'name': 'tensorflow'}]}}},\n",
       "   'Worker': {'replicas': 2,\n",
       "    'restartPolicy': 'Never',\n",
       "    'template': {'spec': {'containers': [{'command': ['python',\n",
       "         '/var/tf_mnist/mnist_with_summaries.py',\n",
       "         '--log_dir=/train/logs',\n",
       "         '--learning_rate=0.01',\n",
       "         '--batch_size=150'],\n",
       "        'image': 'gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0',\n",
       "        'name': 'tensorflow'}]}}}}},\n",
       " 'status': {'conditions': [{'lastTransitionTime': '2021-10-02T19:02:08Z',\n",
       "    'lastUpdateTime': '2021-10-02T19:02:08Z',\n",
       "    'message': 'TFJob mnist is created.',\n",
       "    'reason': 'TFJobCreated',\n",
       "    'status': 'True',\n",
       "    'type': 'Created'},\n",
       "   {'lastTransitionTime': '2021-10-02T19:02:10Z',\n",
       "    'lastUpdateTime': '2021-10-02T19:02:10Z',\n",
       "    'message': 'TFJob default/mnist is running.',\n",
       "    'reason': 'TFJobRunning',\n",
       "    'status': 'True',\n",
       "    'type': 'Running'}],\n",
       "  'replicaStatuses': {'Chief': {'active': 1},\n",
       "   'PS': {'active': 1},\n",
       "   'Worker': {}},\n",
       "  'startTime': '2021-10-02T19:02:09Z'}}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfjob_client.get('mnist', namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the TFJob status, check if the TFJob has been started."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Running'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfjob_client.get_job_status('mnist', namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for the specified job to finish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME                           STATE                TIME                          \n",
      "mnist                          Running              2021-10-02T19:02:10Z          \n",
      "mnist                          Running              2021-10-02T19:02:10Z          \n",
      "mnist                          Running              2021-10-02T19:02:10Z          \n",
      "mnist                          Succeeded            2021-10-02T19:04:10Z          \n"
     ]
    }
   ],
   "source": [
    "tfjob_client.wait_for_job('mnist', namespace=namespace, watch=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check if the TFJob succeeded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfjob_client.is_job_succeeded('mnist', namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the TFJob training logs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The logs of Pod mnist-chief-0:\n",
      " WARNING:tensorflow:From /var/tf_mnist/mnist_with_summaries.py:39: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:252: wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use urllib or similar directly.\n",
      "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: __init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "2021-10-02 19:02:25.434889: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/train-labels-idx1-ubyte.gz\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-labels-idx1-ubyte.gz\n",
      "Accuracy at step 0: 0.1348\n",
      "Accuracy at step 10: 0.787\n",
      "Accuracy at step 20: 0.8648\n",
      "Accuracy at step 30: 0.9056\n",
      "Accuracy at step 40: 0.9162\n",
      "Accuracy at step 50: 0.9237\n",
      "Accuracy at step 60: 0.926\n",
      "Accuracy at step 70: 0.9365\n",
      "Accuracy at step 80: 0.9371\n",
      "Accuracy at step 90: 0.9352\n",
      "Adding run metadata for 99\n",
      "Accuracy at step 100: 0.9439\n",
      "Accuracy at step 110: 0.9434\n",
      "Accuracy at step 120: 0.9382\n",
      "Accuracy at step 130: 0.9444\n",
      "Accuracy at step 140: 0.9487\n",
      "Accuracy at step 150: 0.9462\n",
      "Accuracy at step 160: 0.9454\n",
      "Accuracy at step 170: 0.9426\n",
      "Accuracy at step 180: 0.9473\n",
      "Accuracy at step 190: 0.9536\n",
      "Adding run metadata for 199\n",
      "Accuracy at step 200: 0.9559\n",
      "Accuracy at step 210: 0.9519\n",
      "Accuracy at step 220: 0.9485\n",
      "Accuracy at step 230: 0.95\n",
      "Accuracy at step 240: 0.9563\n",
      "Accuracy at step 250: 0.9575\n",
      "Accuracy at step 260: 0.9591\n",
      "Accuracy at step 270: 0.9589\n",
      "Accuracy at step 280: 0.957\n",
      "Accuracy at step 290: 0.9581\n",
      "Adding run metadata for 299\n",
      "Accuracy at step 300: 0.9606\n",
      "Accuracy at step 310: 0.9585\n",
      "Accuracy at step 320: 0.9593\n",
      "Accuracy at step 330: 0.958\n",
      "Accuracy at step 340: 0.9537\n",
      "Accuracy at step 350: 0.961\n",
      "Accuracy at step 360: 0.9615\n",
      "Accuracy at step 370: 0.962\n",
      "Accuracy at step 380: 0.956\n",
      "Accuracy at step 390: 0.9591\n",
      "Adding run metadata for 399\n",
      "Accuracy at step 400: 0.9554\n",
      "Accuracy at step 410: 0.9604\n",
      "Accuracy at step 420: 0.9638\n",
      "Accuracy at step 430: 0.9614\n",
      "Accuracy at step 440: 0.9645\n",
      "Accuracy at step 450: 0.9683\n",
      "Accuracy at step 460: 0.9591\n",
      "Accuracy at step 470: 0.9645\n",
      "Accuracy at step 480: 0.9557\n",
      "Accuracy at step 490: 0.9647\n",
      "Adding run metadata for 499\n",
      "Accuracy at step 500: 0.9611\n",
      "Accuracy at step 510: 0.9623\n",
      "Accuracy at step 520: 0.9606\n",
      "Accuracy at step 530: 0.9661\n",
      "Accuracy at step 540: 0.9684\n",
      "Accuracy at step 550: 0.9629\n",
      "Accuracy at step 560: 0.9605\n",
      "Accuracy at step 570: 0.9672\n",
      "Accuracy at step 580: 0.9712\n",
      "Accuracy at step 590: 0.9649\n",
      "Adding run metadata for 599\n",
      "Accuracy at step 600: 0.9679\n",
      "Accuracy at step 610: 0.9689\n",
      "Accuracy at step 620: 0.9664\n",
      "Accuracy at step 630: 0.9667\n",
      "Accuracy at step 640: 0.9644\n",
      "Accuracy at step 650: 0.9721\n",
      "Accuracy at step 660: 0.965\n",
      "Accuracy at step 670: 0.9646\n",
      "Accuracy at step 680: 0.9661\n",
      "Accuracy at step 690: 0.9623\n",
      "Adding run metadata for 699\n",
      "Accuracy at step 700: 0.9581\n",
      "Accuracy at step 710: 0.9649\n",
      "Accuracy at step 720: 0.9633\n",
      "Accuracy at step 730: 0.9659\n",
      "Accuracy at step 740: 0.9607\n",
      "Accuracy at step 750: 0.9676\n",
      "Accuracy at step 760: 0.9697\n",
      "Accuracy at step 770: 0.9662\n",
      "Accuracy at step 780: 0.9659\n",
      "Accuracy at step 790: 0.9633\n",
      "Adding run metadata for 799\n",
      "Accuracy at step 800: 0.9638\n",
      "Accuracy at step 810: 0.9592\n",
      "Accuracy at step 820: 0.9642\n",
      "Accuracy at step 830: 0.9682\n",
      "Accuracy at step 840: 0.9695\n",
      "Accuracy at step 850: 0.9657\n",
      "Accuracy at step 860: 0.9696\n",
      "Accuracy at step 870: 0.9695\n",
      "Accuracy at step 880: 0.9711\n",
      "Accuracy at step 890: 0.9687\n",
      "Adding run metadata for 899\n",
      "Accuracy at step 900: 0.9689\n",
      "Accuracy at step 910: 0.9699\n",
      "Accuracy at step 920: 0.9677\n",
      "Accuracy at step 930: 0.9689\n",
      "Accuracy at step 940: 0.9702\n",
      "Accuracy at step 950: 0.9716\n",
      "Accuracy at step 960: 0.9692\n",
      "Accuracy at step 970: 0.967\n",
      "Accuracy at step 980: 0.9687\n",
      "Accuracy at step 990: 0.9665\n",
      "Adding run metadata for 999\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tfjob_client.get_logs('mnist', namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Delete the TFJob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'kind': 'Status',\n",
       " 'apiVersion': 'v1',\n",
       " 'metadata': {},\n",
       " 'status': 'Success',\n",
       " 'details': {'name': 'mnist',\n",
       "  'group': 'kubeflow.org',\n",
       "  'kind': 'tfjobs',\n",
       "  'uid': '4a0b9764-b5c4-4d30-95c3-d3c56d342803'}}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfjob_client.delete('mnist', namespace=namespace)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}