dvc-3-automate-experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Install and init DVC\n",
    "\n",
    "Prerequisites: \n",
    "-  DVC and requirements.txt packages installed (if not - check README.md file for instructions)\n",
    "-  A project repository is a Git repo \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checkout branch `tutorial`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T07:32:19.401395Z",
     "start_time": "2020-07-01T07:32:19.271265Z"
    }
   },
   "source": [
    "```bash\n",
    "git checkout -b dvc-tutorial\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-08T11:18:29.199273Z",
     "start_time": "2019-06-08T11:18:29.196865Z"
    }
   },
   "source": [
    "## Initialize DVC\n",
    "\n",
    "References: \n",
    "- https://dvc.org/doc/get-started/initialize "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T07:32:22.463407Z",
     "start_time": "2020-07-01T07:32:21.450728Z"
    }
   },
   "source": [
    "```bash\n",
    "dvc init\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Commit changes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T07:32:26.446894Z",
     "start_time": "2020-07-01T07:32:26.392814Z"
    }
   },
   "source": [
    "```bash\n",
    "\n",
    "git add .\n",
    "git commit -m \"Initialize DVC\"\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build automated pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create `data_load` stage\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:35.023136Z",
     "start_time": "2020-07-03T19:30:34.904974Z"
    }
   },
   "source": [
    "```bash\n",
    "# Create `data` directory\n",
    "\n",
    "mkdir -p data\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T09:58:42.844179Z",
     "start_time": "2020-10-21T09:58:42.840016Z"
    }
   },
   "source": [
    "```bash\n",
    "# Create data_load pipeline stage\n",
    "\n",
    "dvc run -n data_load \\\n",
    "    -d src/data_load.py \\\n",
    "    -o data/iris.csv \\\n",
    "    -o data/classes.json \\\n",
    "    -p data_load \\\n",
    "    python src/data_load.py \\\n",
    "        --config=params.yaml\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T13:42:01.176530Z",
     "start_time": "2020-10-21T13:42:01.147399Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.0K\tdata/classes.json\n",
      "4.0K\tdata/iris.csv\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "\n",
    "du -sh data/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T10:03:42.595865Z",
     "start_time": "2020-10-21T10:03:42.471874Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m.\u001b[00m\r\n",
      "├── README.md\r\n",
      "├── dvc-3-automate-experiments.ipynb\r\n",
      "├── params.yaml\r\n",
      "├── requirements.txt\r\n",
      "└── \u001b[01;34msrc\u001b[00m\r\n",
      "    ├── __init__.py\r\n",
      "    ├── data_load.py\r\n",
      "    ├── evaluate.py\r\n",
      "    ├── featurization.py\r\n",
      "    ├── split_dataset.py\r\n",
      "    └── train.py\r\n",
      "\r\n",
      "1 directory, 10 files\r\n"
     ]
    }
   ],
   "source": [
    "# Note: we use `tree -I ...` pattern to not list those files that match the wild-card pattern.\n",
    "\n",
    "!tree -I dvc-venv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## dvc.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:37.727096Z",
     "start_time": "2020-07-03T19:30:37.609182Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stages:\r\n",
      "  data_load:\r\n",
      "    cmd: python src/data_load.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - src/data_load.py\r\n",
      "    params:\r\n",
      "    - data_load\r\n",
      "    outs:\r\n",
      "    - data/classes.json\r\n",
      "    - data/iris.csv\r\n"
     ]
    }
   ],
   "source": [
    "!cat dvc.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## params.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:37.877998Z",
     "start_time": "2020-07-03T19:30:37.755666Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r\n",
      "data_load:\r\n",
      "  raw_data_path: data/iris.csv\r\n",
      "  classes_names_path: data/classes.json\r\n",
      "\r\n",
      "featurize:\r\n",
      "  features_path: data/iris_featurized.csv\r\n",
      "  target_column: target\r\n",
      "\r\n",
      "\r\n",
      "data_split:\r\n",
      "  test_size: 0.2\r\n",
      "  train_path: data/train.csv\r\n",
      "  test_path: data/test.csv\r\n",
      "\r\n",
      "\r\n",
      "train:\r\n",
      "  model_path: data/model.joblib\r\n",
      "\r\n",
      "\r\n",
      "evaluate:\r\n",
      "  metrics_file: data/metrics.json\r\n",
      "  confusion_matrix: data/cm.csv\r\n"
     ]
    }
   ],
   "source": [
    "!cat params.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reproduce a pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:39.781553Z",
     "start_time": "2020-07-03T19:30:37.923002Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 'data_load' is cached - skipping run, checking out outputs      core\u001b[39m>\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!dvc repro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Change params.yaml and reproduce \n",
    "\n",
    "Add a new line into `data_load` section:\n",
    "    `dummy_param: dummy_value`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:41.698409Z",
     "start_time": "2020-07-03T19:30:39.807607Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running stage 'data_load' with command:                               core\u001b[39m>\n",
      "\tpython src/data_load.py --config=params.yaml\n",
      "Updating lock file 'dvc.lock'                                         core\u001b[39m>\n",
      "\n",
      "To track the changes with git, run:\n",
      "\n",
      "\tgit add dvc.lock\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!dvc repro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build end-to-end Machine Learning pipeline\n",
    "Stages \n",
    "- extract features \n",
    "- split dataset \n",
    "- train \n",
    "- evaluate \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add feature extraction stage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:45.387596Z",
     "start_time": "2020-07-03T19:30:43.388868Z"
    }
   },
   "source": [
    "```bash\n",
    "\n",
    "dvc run -n feature_extraction \\\n",
    "    -d src/featurization.py \\\n",
    "    -d data/iris.csv \\\n",
    "    -o data/iris_featurized.csv \\\n",
    "    -p data_load,featurize \\\n",
    "    python src/featurization.py \\\n",
    "        --config=params.yaml\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T14:22:48.664322Z",
     "start_time": "2020-10-21T14:22:48.539481Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "README.md                        dvc.yaml\r\n",
      "\u001b[1m\u001b[36mdata\u001b[m\u001b[m                             params.yaml\r\n",
      "dvc-3-automate-experiments.ipynb requirements.txt\r\n",
      "\u001b[1m\u001b[36mdvc-venv\u001b[m\u001b[m                         \u001b[1m\u001b[36msrc\u001b[m\u001b[m\r\n",
      "dvc.lock\r\n"
     ]
    }
   ],
   "source": [
    "!ls "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T14:22:55.648732Z",
     "start_time": "2020-10-21T14:22:55.526727Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stages:\r\n",
      "  data_load:\r\n",
      "    cmd: python src/data_load.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - src/data_load.py\r\n",
      "    params:\r\n",
      "    - data_load\r\n",
      "    outs:\r\n",
      "    - data/classes.json\r\n",
      "    - data/iris.csv\r\n",
      "  feature_extraction:\r\n",
      "    cmd: python src/featurization.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - data/iris.csv\r\n",
      "    - src/featurization.py\r\n",
      "    params:\r\n",
      "    - data_load\r\n",
      "    - featurize\r\n",
      "    outs:\r\n",
      "    - data/iris_featurized.csv\r\n"
     ]
    }
   ],
   "source": [
    "!cat dvc.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T14:22:58.124155Z",
     "start_time": "2020-10-21T14:22:57.646371Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal_length</th>\n",
       "      <th>sepal_width</th>\n",
       "      <th>petal_length</th>\n",
       "      <th>petal_width</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sepal_length  sepal_width  petal_length  petal_width  target\n",
       "0           5.1          3.5           1.4          0.2       0\n",
       "1           4.9          3.0           1.4          0.2       0\n",
       "2           4.7          3.2           1.3          0.2       0\n",
       "3           4.6          3.1           1.5          0.2       0\n",
       "4           5.0          3.6           1.4          0.2       0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "features = pd.read_csv('data/iris_featurized.csv')\n",
    "features.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Commit changes "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T12:58:37.280454Z",
     "start_time": "2020-10-21T12:58:37.272728Z"
    }
   },
   "source": [
    "```bash\n",
    "# Check Git status\n",
    "\n",
    "git status -s\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:45.961182Z",
     "start_time": "2020-07-03T19:30:45.916816Z"
    }
   },
   "source": [
    "```bash\n",
    "# Commit changes \n",
    "\n",
    "git add .\n",
    "git commit -m \"Add stage features_extraction\"\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add split train/test stage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:48.044867Z",
     "start_time": "2020-07-03T19:30:45.984594Z"
    }
   },
   "source": [
    "```bash\n",
    "\n",
    "dvc run -n split_dataset \\\n",
    "    -d src/split_dataset.py \\\n",
    "    -d data/iris_featurized.csv \\\n",
    "    -o data/train.csv \\\n",
    "    -o data/test.csv \\\n",
    "    -p featurize,data_split \\\n",
    "        python src/split_dataset.py \\\n",
    "            --config=params.yaml\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:48.250249Z",
     "start_time": "2020-07-03T19:30:48.209429Z"
    }
   },
   "source": [
    "```bash\n",
    "# Commit changes\n",
    "\n",
    "git add .\n",
    "git commit -m \"Add stage split_dataset\"\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T14:23:25.865051Z",
     "start_time": "2020-10-21T14:23:25.749060Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stages:\r\n",
      "  data_load:\r\n",
      "    cmd: python src/data_load.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - src/data_load.py\r\n",
      "    params:\r\n",
      "    - data_load\r\n",
      "    outs:\r\n",
      "    - data/classes.json\r\n",
      "    - data/iris.csv\r\n",
      "  feature_extraction:\r\n",
      "    cmd: python src/featurization.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - data/iris.csv\r\n",
      "    - src/featurization.py\r\n",
      "    params:\r\n",
      "    - data_load\r\n",
      "    - featurize\r\n",
      "    outs:\r\n",
      "    - data/iris_featurized.csv\r\n",
      "  split_dataset:\r\n",
      "    cmd: python src/split_dataset.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - data/iris_featurized.csv\r\n",
      "    - src/split_dataset.py\r\n",
      "    params:\r\n",
      "    - data_split\r\n",
      "    - featurize\r\n",
      "    outs:\r\n",
      "    - data/test.csv\r\n",
      "    - data/train.csv\r\n"
     ]
    }
   ],
   "source": [
    "!cat dvc.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add train stage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:50.298161Z",
     "start_time": "2020-07-03T19:30:48.275068Z"
    }
   },
   "source": [
    "```bash\n",
    "\n",
    "dvc run -n train \\\n",
    "    -d src/train.py \\\n",
    "    -d data/train.csv \\\n",
    "    -o data/model.joblib \\\n",
    "    -p data_split,train \\\n",
    "        python src/train.py \\\n",
    "            --config=params.yaml\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:50.512656Z",
     "start_time": "2020-07-03T19:30:50.468759Z"
    }
   },
   "source": [
    "```bash\n",
    "# Commit changes\n",
    "\n",
    "git add .\n",
    "git commit -m \"Add stage train\"\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T14:23:55.190584Z",
     "start_time": "2020-10-21T14:23:55.074531Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stages:\r\n",
      "  data_load:\r\n",
      "    cmd: python src/data_load.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - src/data_load.py\r\n",
      "    params:\r\n",
      "    - data_load\r\n",
      "    outs:\r\n",
      "    - data/classes.json\r\n",
      "    - data/iris.csv\r\n",
      "  feature_extraction:\r\n",
      "    cmd: python src/featurization.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - data/iris.csv\r\n",
      "    - src/featurization.py\r\n",
      "    params:\r\n",
      "    - data_load\r\n",
      "    - featurize\r\n",
      "    outs:\r\n",
      "    - data/iris_featurized.csv\r\n",
      "  split_dataset:\r\n",
      "    cmd: python src/split_dataset.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - data/iris_featurized.csv\r\n",
      "    - src/split_dataset.py\r\n",
      "    params:\r\n",
      "    - data_split\r\n",
      "    - featurize\r\n",
      "    outs:\r\n",
      "    - data/test.csv\r\n",
      "    - data/train.csv\r\n",
      "  train:\r\n",
      "    cmd: python src/train.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - data/train.csv\r\n",
      "    - src/train.py\r\n",
      "    params:\r\n",
      "    - data_split\r\n",
      "    - train\r\n",
      "    outs:\r\n",
      "    - data/model.joblib\r\n"
     ]
    }
   ],
   "source": [
    "!cat dvc.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add evaluate stage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T13:03:19.945663Z",
     "start_time": "2020-10-21T13:03:19.941005Z"
    }
   },
   "source": [
    "```bash\n",
    "\n",
    "dvc run -n evaluate \\\n",
    "    -d src/evaluate.py \\\n",
    "    -d data/test.csv \\\n",
    "    -d data/model.joblib \\\n",
    "    -d data/classes.json \\\n",
    "    -m data/metrics.json \\\n",
    "    --plots data/cm.csv \\\n",
    "    -p data_load,data_split,train,evaluate \\\n",
    "        python src/evaluate.py \\\n",
    "            --config=params.yaml\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:30:52.971253Z",
     "start_time": "2020-07-03T19:30:52.919420Z"
    }
   },
   "source": [
    "```bash\n",
    "# Commit changes\n",
    "\n",
    "git add .\n",
    "git commit -m \"Add stage evaluate\"\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T14:24:17.094698Z",
     "start_time": "2020-10-21T14:24:16.977100Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stages:\r\n",
      "  data_load:\r\n",
      "    cmd: python src/data_load.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - src/data_load.py\r\n",
      "    params:\r\n",
      "    - data_load\r\n",
      "    outs:\r\n",
      "    - data/classes.json\r\n",
      "    - data/iris.csv\r\n",
      "  feature_extraction:\r\n",
      "    cmd: python src/featurization.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - data/iris.csv\r\n",
      "    - src/featurization.py\r\n",
      "    params:\r\n",
      "    - data_load\r\n",
      "    - featurize\r\n",
      "    outs:\r\n",
      "    - data/iris_featurized.csv\r\n",
      "  split_dataset:\r\n",
      "    cmd: python src/split_dataset.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - data/iris_featurized.csv\r\n",
      "    - src/split_dataset.py\r\n",
      "    params:\r\n",
      "    - data_split\r\n",
      "    - featurize\r\n",
      "    outs:\r\n",
      "    - data/test.csv\r\n",
      "    - data/train.csv\r\n",
      "  train:\r\n",
      "    cmd: python src/train.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - data/train.csv\r\n",
      "    - src/train.py\r\n",
      "    params:\r\n",
      "    - data_split\r\n",
      "    - train\r\n",
      "    outs:\r\n",
      "    - data/model.joblib\r\n",
      "  evaluate:\r\n",
      "    cmd: python src/evaluate.py --config=params.yaml\r\n",
      "    deps:\r\n",
      "    - data/classes.json\r\n",
      "    - data/model.joblib\r\n",
      "    - data/test.csv\r",
      "\r\n",
      "    - src/evaluate.py\r\n",
      "    params:\r\n",
      "    - data_load\r\n",
      "    - data_split\r\n",
      "    - evaluate\r\n",
      "    - train\r\n",
      "    metrics:\r\n",
      "    - data/metrics.json\r\n",
      "    plots:\r\n",
      "    - data/cm.csv\r\n"
     ]
    }
   ],
   "source": [
    "!cat dvc.yaml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experimenting with reproducible pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How reproduce experiments?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> The most exciting part of DVC is reproducibility.\n",
    ">> Reproducibility is the time you are getting benefits out of DVC instead of spending time defining the ML pipelines.\n",
    "\n",
    "> DVC tracks all the dependencies, which helps you iterate on ML models faster without thinking what was affected by your last change.\n",
    ">> In order to track all the dependencies, DVC finds and reads ALL the DVC-files in a repository and builds a dependency graph (DAG) based on these files.\n",
    "\n",
    "> This is one of the differences between DVC reproducibility and traditional Makefile-like build automation tools (Make, Maven, Ant, Rakefile etc). It was designed in such a way to localize specification of DAG nodes.\n",
    "If you run repro on any created DVC-file from our repository, nothing happens because nothing was changed in the defined pipeline.\n",
    "\n",
    "(c) dvc.org https://dvc.org/doc/tutorial/reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:31:02.889684Z",
     "start_time": "2020-07-03T19:31:00.936546Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 'data_load' didn't change, skipping                             core\u001b[39m>\n",
      "Stage 'feature_extraction' didn't change, skipping\n",
      "Stage 'split_dataset' didn't change, skipping\n",
      "Stage 'train' didn't change, skipping\n",
      "Stage 'evaluate' didn't change, skipping\n",
      "Data and pipelines are up to date.\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Nothing to reproduce\n",
    "\n",
    "!dvc repro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 1: Add features\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create new experiment branch\n",
    "\n",
    "Before editing the code/featurization.py file, please create and checkout a new branch __ratio_features__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:31:05.089755Z",
     "start_time": "2020-07-03T19:31:04.832150Z"
    }
   },
   "source": [
    "```bash\n",
    "# Create new branch\n",
    "\n",
    "git checkout -b exp1-ratio-features\n",
    "git branch\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Update featurization.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "in file __featurization.py__  in function`get_features()` after line \n",
    "\n",
    "```python\n",
    "    features = dataset.copy()\n",
    "```\n",
    "\n",
    "add lines:\n",
    "\n",
    "```python\n",
    "    features['sepal_length_to_sepal_width'] = features['sepal_length'] / features['sepal_width']\n",
    "    features['petal_length_to_petal_width'] = features['petal_length'] / features['petal_width']\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reproduce pipeline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T14:26:16.419175Z",
     "start_time": "2020-10-21T14:26:13.569917Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 'data_load' didn't change, skipping                             core\u001b[39m>\n",
      "Stage 'feature_extraction' is cached - skipping run, checking out outputs\n",
      "Updating lock file 'dvc.lock'\n",
      "\n",
      "Stage 'split_dataset' is cached - skipping run, checking out outputs\n",
      "Updating lock file 'dvc.lock'\n",
      "\n",
      "Stage 'train' is cached - skipping run, checking out outputs\n",
      "Updating lock file 'dvc.lock'\n",
      "\n",
      "Stage 'evaluate' is cached - skipping run, checking out outputs\n",
      "Updating lock file 'dvc.lock'\n",
      "\n",
      "To track the changes with git, run:\n",
      "\n",
      "\tgit add dvc.lock\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!dvc repro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T14:26:17.593890Z",
     "start_time": "2020-10-21T14:26:17.580883Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal_length</th>\n",
       "      <th>sepal_width</th>\n",
       "      <th>petal_length</th>\n",
       "      <th>petal_width</th>\n",
       "      <th>target</th>\n",
       "      <th>sepal_length_to_sepal_width</th>\n",
       "      <th>petal_length_to_petal_width</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.457143</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.633333</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.468750</td>\n",
       "      <td>6.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.483871</td>\n",
       "      <td>7.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.388889</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sepal_length  sepal_width  petal_length  petal_width  target  \\\n",
       "0           5.1          3.5           1.4          0.2       0   \n",
       "1           4.9          3.0           1.4          0.2       0   \n",
       "2           4.7          3.2           1.3          0.2       0   \n",
       "3           4.6          3.1           1.5          0.2       0   \n",
       "4           5.0          3.6           1.4          0.2       0   \n",
       "\n",
       "   sepal_length_to_sepal_width  petal_length_to_petal_width  \n",
       "0                     1.457143                          7.0  \n",
       "1                     1.633333                          7.0  \n",
       "2                     1.468750                          6.5  \n",
       "3                     1.483871                          7.5  \n",
       "4                     1.388889                          7.0  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check features used in this pipeline\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "features = pd.read_csv('data/iris_featurized.csv')\n",
    "features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T14:26:20.852572Z",
     "start_time": "2020-10-21T14:26:20.724853Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On branch exp1-ratio-features\r\n",
      "Changes not staged for commit:\r\n",
      "  (use \"git add <file>...\" to update what will be committed)\r\n",
      "  (use \"git restore <file>...\" to discard changes in working directory)\r\n",
      "\t\u001b[31mmodified:   dvc-3-automate-experiments.ipynb\u001b[m\r\n",
      "\t\u001b[31mmodified:   dvc.lock\u001b[m\r\n",
      "\t\u001b[31mmodified:   src/featurization.py\u001b[m\r\n",
      "\r\n",
      "no changes added to commit (use \"git add\" and/or \"git commit -a\")\r\n"
     ]
    }
   ],
   "source": [
    "!git status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T14:26:28.905617Z",
     "start_time": "2020-10-21T14:26:26.333849Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Path               Metric    Old      New      Change                 core\u001b[39m>\n",
      "data/metrics.json  f1_score  0.15385  0.15385  0.0\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Get difference with metric from previous pipeline\n",
    "!dvc metrics diff --all"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Commit the experiment changes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:31:39.838836Z",
     "start_time": "2020-07-03T19:31:39.445353Z"
    }
   },
   "source": [
    "```bash\n",
    "# Commit changes\n",
    "\n",
    "git add .\n",
    "git commit -m \"Experiment with new features\"\n",
    "git tag -a \"exp1_ratio_features\" -m \"Experiment with new features\"\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 2: Tune Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a new experiment branch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:32:43.387938Z",
     "start_time": "2020-07-03T19:32:43.131917Z"
    }
   },
   "source": [
    "```bash\n",
    "# Create new branch for experiment\n",
    "\n",
    "git checkout -b exp2-tuning-logreg\n",
    "git branch\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:32:52.254763Z",
     "start_time": "2020-07-03T19:32:50.225661Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 'data_load' didn't change, skipping                             core\u001b[39m>\n",
      "Stage 'feature_extraction' didn't change, skipping\n",
      "Stage 'split_dataset' didn't change, skipping\n",
      "Stage 'train' didn't change, skipping\n",
      "Stage 'evaluate' didn't change, skipping\n",
      "Data and pipelines are up to date.\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Nothing to reproduce since code was checked out by `git checkout`\n",
    "# and data files were checked out by `dvc checkout`\n",
    "!dvc repro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuning parameters\n",
    "\n",
    "in file __train.py__ :\n",
    "\n",
    "replace LogisticRegression params with:\n",
    "\n",
    "```python\n",
    "    clf = LogisticRegression(C=0.01, solver='lbfgs', multi_class='multinomial', max_iter=100)\n",
    "```\n",
    "__Note__: here we changed logistic regresssion hyperparameters: C  to 0.1\n",
    "\n",
    "\n",
    "https://dvc.org/doc/tutorials/get-started/experiments#tuning-parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reproduce pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-21T14:27:55.449989Z",
     "start_time": "2020-10-21T14:27:50.294369Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 'data_load' didn't change, skipping                             core\u001b[39m>\n",
      "Stage 'feature_extraction' is cached - skipping run, checking out outputs\n",
      "Updating lock file 'dvc.lock'\n",
      "\n",
      "Stage 'split_dataset' is cached - skipping run, checking out outputs\n",
      "Updating lock file 'dvc.lock'\n",
      "\n",
      "Running stage 'train' with command:\n",
      "\tpython src/train.py --config=params.yaml\n",
      "Updating lock file 'dvc.lock'                                         core\u001b[39m>\n",
      "\n",
      "Running stage 'evaluate' with command:\n",
      "\tpython src/evaluate.py --config=params.yaml\n",
      "Updating lock file 'dvc.lock'                                         core\u001b[39m>\n",
      "\n",
      "To track the changes with git, run:\n",
      "\n",
      "\tgit add dvc.lock\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Re-run pipeline \n",
    "\n",
    "!dvc repro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:03:21.593735Z",
     "start_time": "2020-10-22T14:03:21.473130Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"f1_score\": 0.9305555555555555}"
     ]
    }
   ],
   "source": [
    "# Get difference with metric from previous pipeline\n",
    "\n",
    "!cat data/metrics.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:03:48.640425Z",
     "start_time": "2020-10-22T14:03:47.009948Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tdata/metrics.json:                                                   core\u001b[39m>\n",
      "\t\tf1_score: 0.9305555555555555\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!dvc metrics show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:03:52.823772Z",
     "start_time": "2020-10-22T14:03:50.239522Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Path               Metric    Old      New      Change                 core\u001b[39m>\n",
      "data/metrics.json  f1_score  0.15385  0.93056  0.77671\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!dvc metrics diff --all"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Commit changes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:36:15.808072Z",
     "start_time": "2020-07-03T19:36:15.762972Z"
    },
    "scrolled": true
   },
   "source": [
    "```bash\n",
    "# Commit changes\n",
    "\n",
    "git add .\n",
    "git commit -m \"Tune model. LogisticRegression. C=0.1\"\n",
    "git tag -a \"exp2_tuning_logreg\" -m \"Tune model. LogisticRegression. C=0.01\"\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment 3: Use SVM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:14:01.831192Z",
     "start_time": "2020-07-03T19:14:01.829062Z"
    }
   },
   "source": [
    "### Create a new experiment branch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:36:20.443851Z",
     "start_time": "2020-07-03T19:36:20.187021Z"
    }
   },
   "source": [
    "```bash\n",
    "# Create a new experiment branch \n",
    "\n",
    "git checkout -b exp3-svm\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Update train.py\n",
    "\n",
    "in file __train.py__ replace line\n",
    "\n",
    "```python\n",
    "    clf = LogisticRegression(C=0.1, solver='newton-cg', multi_class='multinomial', max_iter=100)\n",
    "```\n",
    "\n",
    "with line\n",
    "\n",
    "```python\n",
    "    clf = SVC(C=0.01, kernel='linear', gamma='scale', degree=5)\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reproduce pipeline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:36:35.537208Z",
     "start_time": "2020-07-03T19:36:32.544097Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 'data_load' didn't change, skipping                             core\u001b[39m>\n",
      "Stage 'feature_extraction' didn't change, skipping\n",
      "Stage 'split_dataset' didn't change, skipping\n",
      "Running stage 'train' with command:\n",
      "\tpython src/train.py --config=params.yaml\n",
      "Updating lock file 'dvc.lock'                                         core\u001b[39m>\n",
      "\n",
      "Running stage 'evaluate' with command:\n",
      "\tpython src/evaluate.py --config=params.yaml\n",
      "Updating lock file 'dvc.lock'                                         core\u001b[39m>\n",
      "\n",
      "To track the changes with git, run:\n",
      "\n",
      "\tgit add dvc.lock\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!dvc repro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:05:25.052314Z",
     "start_time": "2020-10-22T14:05:23.390114Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tdata/metrics.json:                                                   core\u001b[39m>\n",
      "\t\tf1_score: 0.9665831244778613\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!dvc metrics show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:05:35.904182Z",
     "start_time": "2020-10-22T14:05:35.779856Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On branch exp3-svm\r\n",
      "Changes not staged for commit:\r\n",
      "  (use \"git add <file>...\" to update what will be committed)\r\n",
      "  (use \"git restore <file>...\" to discard changes in working directory)\r\n",
      "\t\u001b[31mmodified:   dvc.lock\u001b[m\r\n",
      "\t\u001b[31mmodified:   src/train.py\u001b[m\r\n",
      "\r\n",
      "no changes added to commit (use \"git add\" and/or \"git commit -a\")\r\n"
     ]
    }
   ],
   "source": [
    "!git status"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Commit changes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:36:41.766798Z",
     "start_time": "2020-07-03T19:36:41.377185Z"
    }
   },
   "source": [
    "```bash\n",
    "# Commit changes\n",
    "\n",
    "git add .\n",
    "git commit -m \"Experiment 3 with SVM estimator\"\n",
    "git tag -a \"exp3_svm\" -m \"Experiment 3 with SVM estimator\"\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge best experiment `dvc-tutorial ` branch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:45:17.537969Z",
     "start_time": "2020-07-03T19:45:17.463715Z"
    },
    "scrolled": true
   },
   "source": [
    "```bash\n",
    "# Merge the best experiment\n",
    "\n",
    "git checkout dvc-tutorial \n",
    "git merge exp3_svm\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare params "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:07:30.189016Z",
     "start_time": "2020-10-22T14:07:27.537384Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0m                                                                  core\u001b[39m>"
     ]
    }
   ],
   "source": [
    "# Get params diffs \n",
    "\n",
    "!dvc params diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:07:47.322051Z",
     "start_time": "2020-10-22T14:07:44.759864Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Path         Param                         Old                       Newre\u001b[39m>\n",
      "params.yaml  data_load.classes_names_path  data/classes.json         data/classes.json\n",
      "params.yaml  data_load.raw_data_path       data/iris.csv             data/iris.csv\n",
      "params.yaml  data_split.test_path          data/test.csv             data/test.csv\n",
      "params.yaml  data_split.test_size          0.2                       0.2\n",
      "params.yaml  data_split.train_path         data/train.csv            data/train.csv\n",
      "params.yaml  evaluate.confusion_matrix     data/cm.csv               data/cm.csv\n",
      "params.yaml  evaluate.metrics_file         data/metrics.json         data/metrics.json\n",
      "params.yaml  featurize.features_path       data/iris_featurized.csv  data/iris_featurized.csv\n",
      "params.yaml  featurize.target_column       target                    target\n",
      "params.yaml  train.model_path              data/model.joblib         data/model.joblib\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Compare parameters with a specific commit, a tag or any revision\n",
    "\n",
    "!dvc params diff --all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:09:20.304575Z",
     "start_time": "2020-07-03T19:09:18.649548Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"params.yaml\": {\"data_split.test_path\": {\"old\": \"data/test.csv\", \"new\": \"data/test.csv\"}, \"featurize.target_column\": {\"old\": \"target\", \"new\": \"target\"}, \"evaluate.confusion_matrix\": {\"old\": \"data/cm.csv\", \"new\": \"data/cm.csv\"}, \"train.model_path\": {\"old\": \"data/model.joblib\", \"new\": \"data/model.joblib\"}, \"data_load.classes_names_path\": {\"old\": \"data/classes.json\", \"new\": \"data/classes.json\"}, \"data_split.test_size\": {\"old\": 0.2, \"new\": 0.2, \"diff\": 0.0}, \"data_load.dummy_param\": {\"old\": \"dummy_value\", \"new\": \"dummy_value\"}, \"data_load.raw_data_path\": {\"old\": \"data/iris.csv\", \"new\": \"data/iris.csv\"}, \"data_split.train_path\": {\"old\": \"data/train.csv\", \"new\": \"data/train.csv\"}, \"evaluate.metrics_file\": {\"old\": \"data/metrics.json\", \"new\": \"data/metrics.json\"}, \"featurize.features_path\": {\"old\": \"data/iris_featurized.csv\", \"new\": \"data/iris_featurized.csv\"}}}\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!dvc params diff --show-json --all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-03T19:09:27.495017Z",
     "start_time": "2020-07-03T19:09:25.848748Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Path        | Param                        | Old                      | New                      |\n",
      "|-------------|------------------------------|--------------------------|--------------------------|\n",
      "| params.yaml | data_load.classes_names_path | data/classes.json        | data/classes.json        |\n",
      "| params.yaml | data_load.dummy_param        | dummy_value              | dummy_value              |\n",
      "| params.yaml | data_load.raw_data_path      | data/iris.csv            | data/iris.csv            |\n",
      "| params.yaml | data_split.test_path         | data/test.csv            | data/test.csv            |\n",
      "| params.yaml | data_split.test_size         | 0.2                      | 0.2                      |\n",
      "| params.yaml | data_split.train_path        | data/train.csv           | data/train.csv           |\n",
      "| params.yaml | evaluate.confusion_matrix    | data/cm.csv              | data/cm.csv              |\n",
      "| params.yaml | evaluate.metrics_file        | data/metrics.json        | data/metrics.json        |\n",
      "| params.yaml | featurize.features_path      | data/iris_featurized.csv | data/iris_featurized.csv |\n",
      "| params.yaml | featurize.target_column      | target                   | target                   |\n",
      "| params.yaml | train.model_path             | data/model.joblib        | data/model.joblib        |\n",
      "\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!dvc params diff --show-md --all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:08:29.319419Z",
     "start_time": "2020-10-22T14:08:29.189441Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mcommit 336832e6c8c51861d58e258b6cf7bc5ddc750459\u001b[m\u001b[33m (\u001b[m\u001b[1;36mHEAD -> \u001b[m\u001b[1;32mdvc-tutorial\u001b[m\u001b[33m, \u001b[m\u001b[1;33mtag: exp3_svm\u001b[m\u001b[33m)\u001b[m\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Thu Oct 22 17:05:44 2020 +0300\r\n",
      "\r\n",
      "    Experiment 3 with SVM estimator\r\n",
      "\r\n",
      "\u001b[33mcommit aff5b7f5d143895108b4dac9939a9c0cd06a349d\u001b[m\u001b[33m (\u001b[m\u001b[1;33mtag: exp2_tuning_logreg\u001b[m\u001b[33m, \u001b[m\u001b[1;32mexp2-tuning-logreg\u001b[m\u001b[33m)\u001b[m\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Thu Oct 22 17:04:33 2020 +0300\r\n",
      "\r\n",
      "    Tune model. LogisticRegression. C=0.1\r\n",
      "\r\n",
      "\u001b[33mcommit 7ab2b518063b63742a396ca83ce6a092a260589a\u001b[m\u001b[33m (\u001b[m\u001b[1;33mtag: exp1_ratio_features\u001b[m\u001b[33m, \u001b[m\u001b[1;32mexp1-ratio-features\u001b[m\u001b[33m)\u001b[m\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Wed Oct 21 17:27:03 2020 +0300\r\n",
      "\r\n",
      "    Experiment with new features\r\n",
      "\r\n",
      "\u001b[33mcommit 7619688214cc3b9fe3d3b59674c07c12fc134b47\u001b[m\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Wed Oct 21 17:24:13 2020 +0300\r\n",
      "\r\n",
      "    Add stage evaluate\r\n",
      "\r\n",
      "\u001b[33mcommit 2a59d083d38b1a15dab6fe4c2b53c00a9d7f0447\u001b[m\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Wed Oct 21 17:23:50 2020 +0300\r\n",
      "\r\n",
      "    Add stage train\r\n",
      "\r\n",
      "\u001b[33mcommit 1a908460f95ed339ccdf735a193054b8af0632dc\u001b[m\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Wed Oct 21 17:23:22 2020 +0300\r\n",
      "\r\n",
      "    Add stage split_dataset\r\n",
      "\r\n",
      "\u001b[33mcommit 915ba326763d905b4c3559cb29ec825f48fb11f0\u001b[m\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Wed Oct 21 17:23:04 2020 +0300\r\n",
      "\r\n",
      "    Add stage features_extraction\r\n",
      "\r\n",
      "\u001b[33mcommit 0e12b6aab49d3d96570f95950df411ccf00c6326\u001b[m\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Wed Oct 21 12:56:45 2020 +0300\r\n",
      "\r\n",
      "    Initialize DVC\r\n",
      "\r\n",
      "\u001b[33mcommit 25cb0004d22fdc923536ac5232e146361e74e90c\u001b[m\u001b[33m (\u001b[m\u001b[1;31morigin/update-dvc-tutorials\u001b[m\u001b[33m, \u001b[m\u001b[1;32mupdate-dvc-tutorials\u001b[m\u001b[33m)\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Mon Oct 19 18:03:15 2020 +0900\r\n",
      "\r\n",
      "    Update tutorial notebook\r\n",
      "\r\n",
      "\u001b[33mcommit bf430d9af143db178d6fa39b405437ee9f8ab1f3\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Mon Oct 19 17:36:07 2020 +0900\r\n",
      "\r\n",
      "    Update .gitignore\r\n",
      "\r\n",
      "\u001b[33mcommit 0ecca2cdab5972ba68f88cc395b2c4ba4dc47dcf\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Mon Oct 19 17:35:42 2020 +0900\r\n",
      "\r\n",
      "    Update documentation\r\n",
      "\r\n",
      "\u001b[33mcommit c6293c84e8aa3b0e65afaaecc1a3c9b03d6b2dec\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Mon Oct 19 17:35:23 2020 +0900\r\n",
      "\r\n",
      "    Update versions of python libraries\r\n",
      "\r\n",
      "\u001b[33mcommit 15bd59fe85e1e002d1ea45230dc61f3b9c4dcfe3\u001b[m\u001b[33m (\u001b[m\u001b[1;31morigin/dev\u001b[m\u001b[33m)\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Tue Aug 18 12:45:16 2020 +0900\r\n",
      "\r\n",
      "    Update DVC -> 1.5.0\r\n",
      "\r\n",
      "\u001b[33mcommit 3573af273082ae1cad1b141131eb824e314eee43\u001b[m\u001b[33m (\u001b[m\u001b[1;32mdev\u001b[m\u001b[33m)\u001b[m\r\n",
      "Merge: aeecba0 1102dc2\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Sat Jul 4 07:23:25 2020 +0300\r\n",
      "\r\n",
      "    Updated evaluate and params\r\n",
      "\r\n",
      "\u001b[33mcommit aeecba0880f016303eaa20181d27b60bd3ceb388\u001b[m\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Sat Jul 4 07:16:31 2020 +0300\r\n",
      "\r\n",
      "    Update experimeting and metrics section\r\n",
      "\r\n",
      "\u001b[33mcommit 1102dc2e3f636b2d37558f95a960c788f3de32ed\u001b[m\r\n",
      "Merge: 855c61a 92ac211\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Wed Jul 1 07:22:32 2020 +0000\r\n",
      "\r\n",
      "    Merge branch 'update-confusion-matrix' into 'dev'\r\n",
      "    \r\n",
      "    update confusion matrix\r\n",
      "    \r\n",
      "    See merge request 7labs.ru/tutorials-dvc/dvc-3-automate-experiments!4\r\n",
      "\r\n",
      "\u001b[33mcommit 92ac211f2139095965d0e26304d2d39003136def\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Tue Jun 30 13:08:30 2020 +0900\r\n",
      "\r\n",
      "    update confusion matrix\r\n",
      "\r\n",
      "\u001b[33mcommit 855c61ac3f02f8938445fe749846e20d01e0f247\u001b[m\r\n",
      "Merge: 22aeb23 7fbf4d8\r\n",
      "Author: Alexander Kolosov <alexckolosov@gmail.com>\r\n",
      "Date:   Mon Jun 29 08:47:37 2020 +0000\r\n",
      "\r\n",
      "    Merge branch 'dev-update-pipelines' into 'dev'\r\n",
      "    \r\n",
      "    Dev update pipelines\r\n",
      "    \r\n",
      "    See merge request 7labs.ru/tutorials-dvc/dvc-3-automate-experiments!3\r\n",
      "\r\n",
      "\u001b[33mcommit 7fbf4d8f4e54be947f77dce09191b4f6fbb287f0\u001b[m\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Mon Jun 29 08:47:37 2020 +0000\r\n",
      "\r\n",
      "    Dev update pipelines\r\n",
      "\r\n",
      "\u001b[33mcommit 22aeb23eb6b54f12f11c76a5714dbf6bff5f11f9\u001b[m\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Sun Jun 28 19:02:29 2020 +0300\r\n",
      "\r\n",
      "    Update name of tutorial and notebook\r\n",
      "\r\n",
      "\u001b[33mcommit 110a584e41fa7c140bbaf8130f70d4112e58d1a4\u001b[m\r\n",
      "Merge: 2d7e834 a8d3200\r\n",
      "Author: Mikhail <mnrozhkov@gmail.com>\r\n",
      "Date:   Sat Jun 27 07:49:11 2020 +0000\r\n",
      "\r\n",
      "    Merge branch 'update-software' into 'dev'\r\n",
      "    \r\n",
      "    Update software\r\n",
      "    \r\n",
      "    See merge request 7labs.ru/tutorials-dvc/dvc-3-automate-experiments!2\r\n",
      "\r\n",
      "\u001b[33mcommit a8d3200b8cbffdc4af1c7204710d217e9f685928\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Fri Jun 26 17:58:32 2020 +0900\r\n",
      "\r\n",
      "    intall toc for jupyter notebook\r\n",
      "\r\n",
      "\u001b[33mcommit 8b042ad196928f9584b4bbce058625896af78d9d\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Fri Jun 26 17:58:12 2020 +0900\r\n",
      "\r\n",
      "    upgrade dvc\r\n",
      "\r\n",
      "\u001b[33mcommit 2d7e834a6d115d1b47253377b3baaace559e3259\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Thu Jun 11 12:53:18 2020 +0900\r\n",
      "\r\n",
      "    add data/ to .gitignore\r\n",
      "\r\n",
      "\u001b[33mcommit 8817b3ed1f82ed1c4feb9122d49237b37356e70e\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Wed Jun 10 22:56:32 2020 +0900\r\n",
      "\r\n",
      "    update Lesson 4.ipynb: append description of dvc plots diff\r\n",
      "\r\n",
      "\u001b[33mcommit a8db726c3f368c39180d61d21f21bf6727db20c0\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Wed Jun 10 22:44:00 2020 +0900\r\n",
      "\r\n",
      "    update Lesson 4.ipynb: add section for dvc metrics diff and dvc plots\r\n",
      "\r\n",
      "\u001b[33mcommit 77559e316fe6b5fd0a11f27a06fbc9eed1c2b606\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Wed Jun 10 22:43:09 2020 +0900\r\n",
      "\r\n",
      "    update src/evaluate.py: put metric and confusion matrix in separated files\r\n",
      "\r\n",
      "\u001b[33mcommit a0afac2ff2dc7c5815c72ec3770888b67e5f04e7\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Wed Jun 10 12:05:51 2020 +0900\r\n",
      "\r\n",
      "    refactor code modules\r\n",
      "\r\n",
      "\u001b[33mcommit 73846297879b1f1be3868c64e73b7d8ad6966b09\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Wed Jun 10 12:04:37 2020 +0900\r\n",
      "\r\n",
      "    fix Lesson 4.ipynb\r\n",
      "\r\n",
      "\u001b[33mcommit b6ba776f8607c6481e34f8a40af4c23a5cd36990\u001b[m\r\n",
      "Author: Alex <alexckolosov@gmail.com>\r\n",
      "Date:   Tue Jun 9 19:27:13 2020 +0900\r\n",
      "\r\n",
      "    create repo structure for  lesson 4\r\n"
     ]
    }
   ],
   "source": [
    "# To see the difference between two specific commits, both need to be specified:\n",
    "\n",
    "!git log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:08:58.251510Z",
     "start_time": "2020-10-22T14:08:55.660152Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0m                                                                  core\u001b[39m>"
     ]
    }
   ],
   "source": [
    "!dvc params diff 7619688214cc3b9fe3d3b59674c07c12fc134b47 HEAD^"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:09:14.039904Z",
     "start_time": "2020-10-22T14:09:12.356063Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tdata/metrics.json:                                                   core\u001b[39m>\n",
      "\t\tf1_score: 0.9665831244778613\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# this pipeline metrics \n",
    "\n",
    "!dvc metrics show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:09:18.038960Z",
     "start_time": "2020-10-22T14:09:16.234043Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dvc-tutorial:                                                         core\u001b[39m>\n",
      "\tdata/metrics.json:\n",
      "\t\tf1_score: 0.9665831244778613\n",
      "exp1-ratio-features:\n",
      "\tdata/metrics.json:\n",
      "\t\tf1_score: 0.15384615384615383\n",
      "exp2-tuning-logreg:\n",
      "\tdata/metrics.json:\n",
      "\t\tf1_score: 0.9305555555555555\n",
      "exp3-svm:\n",
      "\tdata/metrics.json:\n",
      "\t\tf1_score: 0.9665831244778613\n",
      "exp1_ratio_features:\n",
      "\tdata/metrics.json:\n",
      "\t\tf1_score: 0.15384615384615383\n",
      "exp2_tuning_logreg:\n",
      "\tdata/metrics.json:\n",
      "\t\tf1_score: 0.9305555555555555\n",
      "exp3_svm:\n",
      "\tdata/metrics.json:\n",
      "\t\tf1_score: 0.9665831244778613\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# show all commited pipelines metrics (all branch and tags)\n",
    "\n",
    "!dvc metrics show -a -T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare metrics (get differences)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:09:38.852532Z",
     "start_time": "2020-10-22T14:09:36.162510Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0m                                                                  core\u001b[39m>"
     ]
    }
   ],
   "source": [
    "!dvc metrics diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:09:41.557718Z",
     "start_time": "2020-10-22T14:09:38.912543Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Path               Metric    Old      New      Change                 core\u001b[39m>\n",
      "data/metrics.json  f1_score  0.96658  0.96658  0.0\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# --all - list all metrics, even those without changes\n",
    "\n",
    "!dvc metrics diff --all"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* чтобы сравнить текущую метрики из текущего коммита и из другого, нужно указать другой (old) коммит:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:09:57.211470Z",
     "start_time": "2020-10-22T14:09:54.710817Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Path               Metric    Old      New      Change                 core\u001b[39m>\n",
      "data/metrics.json  f1_score  0.15385  0.96658  0.81274\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Compare old and new branches\n",
    "\n",
    "!dvc metrics diff exp1-ratio-features exp3-svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:10:03.187146Z",
     "start_time": "2020-10-22T14:10:00.568957Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Path               Metric    Old      New      Change                 core\u001b[39m>\n",
      "data/metrics.json  f1_score  0.15385  0.96658  0.81274\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Equivalent to `!dvc metrics diff exp1-ratio-features dvc-tutorial`, because dvc-tutorial - current branch\n",
    "\n",
    "!dvc metrics diff exp1-ratio-features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:10:38.911976Z",
     "start_time": "2020-10-22T14:10:36.304792Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Path              | Metric   | Old     | New     | Change   |       core\u001b[39m>\n",
      "|-------------------|----------|---------|---------|----------|\n",
      "| data/metrics.json | f1_score | 0.15385 | 0.96658 | 0.81274  |\n",
      "\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!dvc metrics diff exp1-ratio-features --show-md"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Plots\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:11:04.697127Z",
     "start_time": "2020-10-22T14:11:04.694448Z"
    }
   },
   "outputs": [],
   "source": [
    "from IPython.display import IFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:15:07.599224Z",
     "start_time": "2020-10-22T14:15:05.455604Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file:///Users/mnrozhkov/dev/dvc/dvc-3-automate-experiments/data/plots-show.html\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!dvc plots show  --template confusion \"data/cm.csv\" -x actual -y predicted -o data/plots-show.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:15:07.664691Z",
     "start_time": "2020-10-22T14:15:07.660138Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"800\"\n",
       "            height=\"500\"\n",
       "            src=\"data/plots-show.html\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fe135bfdac8>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IFrame(src='data/plots-show.html', width=800, height=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:15:16.499865Z",
     "start_time": "2020-10-22T14:15:15.626027Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file:///Users/mnrozhkov/dev/dvc/dvc-3-automate-experiments/data/plots-diff.html\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Build metircs plots for all 3 experiments\n",
    "!dvc plots diff -t confusion -o data/plots-diff.html exp1-ratio-features exp3-svm -x predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T14:15:23.321541Z",
     "start_time": "2020-10-22T14:15:23.317724Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"800\"\n",
       "            height=\"500\"\n",
       "            src=\"data/plots-diff.html\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x7fe135bf3b70>"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IFrame(src='data/plots-diff.html', width=800, height=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "230.947px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}