From 9cc440f4d2a236fd441f80645351845174795e61 Mon Sep 17 00:00:00 2001 From: dberenbaum Date: Sat, 6 Jan 2024 09:53:55 -0500 Subject: [PATCH] experiments: make notebook more like the pipeline --- .../code/notebooks/TrainSegModel.ipynb | 81 +++++++++---------- example-get-started-experiments/generate.sh | 4 +- 2 files changed, 38 insertions(+), 47 deletions(-) diff --git a/example-get-started-experiments/code/notebooks/TrainSegModel.ipynb b/example-get-started-experiments/code/notebooks/TrainSegModel.ipynb index 6ebc1a0d..1549f42a 100644 --- a/example-get-started-experiments/code/notebooks/TrainSegModel.ipynb +++ b/example-get-started-experiments/code/notebooks/TrainSegModel.ipynb @@ -6,30 +6,11 @@ "metadata": {}, "outputs": [], "source": [ - "from pathlib import Path\n", - "\n", - "ROOT = Path(\"../\")\n", - "DATA = ROOT / \"data\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import warnings\n", - "warnings.filterwarnings(\"ignore\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "import os\n", "import shutil\n", "from functools import partial\n", + "from pathlib import Path\n", + "import warnings\n", "\n", "import numpy as np\n", "import torch\n", @@ -41,7 +22,10 @@ "from fastai.vision.all import (Resize, SegmentationDataLoaders,\n", " imagenet_stats, models, unet_learner)\n", "from ruamel.yaml import YAML\n", - "from PIL import Image" + "from PIL import Image\n", + "\n", + "os.chdir(\"..\")\n", + "warnings.filterwarnings(\"ignore\")" ] }, { @@ -77,14 +61,14 @@ "source": [ "test_regions = [\"REGION_1-\"]\n", "\n", - "img_fpaths = get_files(DATA / \"pool_data\" / \"images\", extensions=\".jpg\")\n", + "img_fpaths = get_files(Path(\"data\") / \"pool_data\" / \"images\", extensions=\".jpg\")\n", "\n", - "train_data_dir = DATA / \"train_data\"\n", + "train_data_dir = Path(\"data\") / \"train_data\"\n", "train_data_dir.mkdir(exist_ok=True)\n", - "test_data_dir = DATA / \"test_data\"\n", + "test_data_dir = Path(\"data\") / \"test_data\"\n", "test_data_dir.mkdir(exist_ok=True)\n", "for img_path in img_fpaths:\n", - " msk_path = DATA / \"pool_data\" / \"masks\" / f\"{img_path.stem}.png\"\n", + " msk_path = Path(\"data\") / \"pool_data\" / \"masks\" / f\"{img_path.stem}.png\"\n", " if any(region in str(img_path) for region in test_regions):\n", " shutil.copy(img_path, test_data_dir)\n", " shutil.copy(msk_path, test_data_dir)\n", @@ -182,44 +166,41 @@ " dice_list.append(dice)\n", " return np.mean(dice_list)\n", "\n", + "\n", "def evaluate(learn):\n", - " test_img_fpaths = sorted(get_files(DATA / \"test_data\", extensions=\".jpg\"))\n", + " test_img_fpaths = sorted(get_files(Path(\"data\") / \"test_data\", extensions=\".jpg\"))\n", " test_dl = learn.dls.test_dl(test_img_fpaths)\n", " preds, _ = learn.get_preds(dl=test_dl)\n", " masks_pred = np.array(preds[:, 1, :] > 0.5, dtype=np.uint8)\n", " test_mask_fpaths = [\n", - " get_mask_path(fpath, DATA / \"test_data\") for fpath in test_img_fpaths\n", + " get_mask_path(fpath, Path(\"data\") / \"test_data\") for fpath in test_img_fpaths\n", " ]\n", " masks_true = [Image.open(mask_path) for mask_path in test_mask_fpaths]\n", + "\n", " dice_multi = 0.0\n", " for ii in range(len(masks_true)):\n", " mask_pred, mask_true = masks_pred[ii], masks_true[ii]\n", - " width, height = mask_true.shape[1], mask_true.shape[0]\n", " mask_pred = np.array(\n", - " Image.fromarray(mask_pred).resize((width, height)),\n", + " Image.fromarray(mask_pred).resize((mask_true.shape[1], mask_true.shape[0])),\n", " dtype=int\n", " )\n", " mask_true = np.array(mask_true, dtype=int)\n", " dice_multi += dice(mask_true, mask_pred) / len(masks_true)\n", + "\n", " return dice_multi" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "train_arch = 'shufflenet_v2_x2_0'\n", - "models_dir = ROOT / \"models\"\n", - "models_dir.mkdir(exist_ok=True)\n", - "results_dir = ROOT / \"results\" / \"train\"\n", "\n", "for base_lr in [0.001, 0.005, 0.01]:\n", " # initialize dvclive, optionally provide output path, and save results as a dvc experiment\n", - " with Live(str(results_dir), save_dvc_exp=True, report=\"notebook\") as live:\n", + " with Live(\"results/train\", save_dvc_exp=False, report=\"notebook\") as live:\n", " # log a parameter\n", " live.log_param(\"train_arch\", train_arch)\n", " fine_tune_args = {\n", @@ -237,19 +218,22 @@ " **fine_tune_args,\n", " cbs=[DVCLiveCallback(live=live)])\n", "\n", - " learn.export(fname=(models_dir / \"model.pkl\").absolute())\n", - "\n", - " # add additional post-training summary metrics\n", - " live.summary[\"evaluate/dice_multi\"] = evaluate(learn)\n", - "\n", " # save model artifact to dvc\n", + " models_dir = Path(\"models\")\n", + " models_dir.mkdir(exist_ok=True)\n", + " learn.export(fname=(models_dir / \"model.pkl\").absolute())\n", + " torch.save(learn.model, (models_dir / \"model.pth\").absolute())\n", " live.log_artifact(\n", " str(models_dir / \"model.pkl\"),\n", " type=\"model\",\n", " name=\"pool-segmentation\",\n", " desc=\"This is a Computer Vision (CV) model that's segmenting out swimming pools from satellite images.\",\n", " labels=[\"cv\", \"segmentation\", \"satellite-images\", \"unet\"],\n", - " )\n" + " )\n", + "\n", + " # add additional post-training summary metrics\n", + " with Live(\"results/evaluate\") as live:\n", + " live.summary[\"dice_multi\"] = evaluate(learn)" ] }, { @@ -280,6 +264,13 @@ "source": [ "learn.show_results(max_n=6, alpha=0.7)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -298,7 +289,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.6" + "version": "3.11.6" }, "vscode": { "interpreter": { diff --git a/example-get-started-experiments/generate.sh b/example-get-started-experiments/generate.sh index 49ff0686..26982658 100755 --- a/example-get-started-experiments/generate.sh +++ b/example-get-started-experiments/generate.sh @@ -82,7 +82,7 @@ pip install -r requirements.txt --extra-index-url https://download.pytorch.org/w pip install jupyter jupyter nbconvert --execute 'notebooks/TrainSegModel.ipynb' --inplace # Apply best experiment -BEST_EXP_ROW=$(dvc exp show --drop '.*' --keep 'Experiment|evaluate/dice_multi|base_lr' --csv --sort-by evaluate/dice_multi | tail -n 1) +BEST_EXP_ROW=$(dvc exp show --drop '.*' --keep 'Experiment|results/evaluate/metrics.json:dice_multi|base_lr' --csv --sort-by 'results/evaluate/metrics.json:dice_multi' | tail -n 1) BEST_EXP_NAME=$(echo $BEST_EXP_ROW | cut -d, -f 1) BEST_EXP_BASE_LR=$(echo $BEST_EXP_ROW | cut -d, -f 3) dvc exp apply $BEST_EXP_NAME @@ -98,7 +98,7 @@ cp $HERE/code/params.yaml . sed -e "s/base_lr: 0.01/base_lr: $BEST_EXP_BASE_LR/" -i".bkp" params.yaml rm params.yaml.bkp -git rm -r --cached 'results' +git rm -r --cached 'results' 'models' git commit -m "stop tracking results" dvc stage add -n data_split \