From 6caa78ce06d131f82ab04f4c2fd5aabd058a359a Mon Sep 17 00:00:00 2001 From: Lars Voegtlin Date: Mon, 29 Nov 2021 14:50:11 +0100 Subject: [PATCH 1/4] :wrench: removed the hydra config and replaced it with a mode config. two modes: dev and exp --- configs/config.yaml | 2 -- configs/experiment/cb55_full_run_unet.yaml | 5 ++++- configs/experiment/cb55_select_train15_unet.yaml | 5 ++++- configs/experiment/cb55_select_train1_val1_unet.yaml | 5 ++++- configs/experiment/dev_rgb_full_page.yaml | 1 + configs/experiment/dev_rgb_full_page_predict.yaml | 1 + configs/experiment/dev_rolf_format.yaml | 1 + configs/experiment/dev_rolf_format_prediction.yaml | 1 + configs/experiment/dev_rotnet_cnn_basic_cb55_10.yaml | 1 + .../dev_rotnet_pt_resnet18_cb55_10_segmentation.yaml | 1 + configs/experiment/dev_rotnet_resnet18_cb55_10.yaml | 1 + configs/experiment/dev_rotnet_resnet50_cb55_10.yaml | 1 + configs/experiment/development_baby_unet_cb55_10.yaml | 1 + configs/experiment/development_baby_unet_rgb_data.yaml | 1 + configs/experiment/rotnet_resnet18_cb55_full.yaml | 5 ++++- .../experiment/rotnet_resnet18_cb55_train10_last.yaml | 5 ++++- .../experiment/rotnet_resnet18_cb55_train19_last.yaml | 5 ++++- .../experiment/synthetic_baby_unet_layoutD_gtD.yaml | 5 ++++- .../synthetic_baby_unet_layoutD_gtD_rolf_format.yaml | 5 ++++- .../experiment/synthetic_baby_unet_layoutR_gtD.yaml | 5 ++++- configs/hydra/default.yaml | 0 configs/mode/development.yaml | 10 ++++++++++ configs/mode/experiment.yaml | 10 ++++++++++ src/utils/utils.py | 5 +++++ 24 files changed, 71 insertions(+), 11 deletions(-) delete mode 100644 configs/hydra/default.yaml create mode 100644 configs/mode/development.yaml create mode 100644 configs/mode/experiment.yaml diff --git a/configs/config.yaml b/configs/config.yaml index 071b481c..fcfe7b08 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -6,8 +6,6 @@ defaults: - experiment: development_baby_unet_cb55_10.yaml - hparams_search: null - - hydra: default.yaml - # enable color logging - override hydra/job_logging: colorlog - override hydra/hydra_logging: colorlog diff --git a/configs/experiment/cb55_full_run_unet.yaml b/configs/experiment/cb55_full_run_unet.yaml index a9114e84..0e4377ae 100644 --- a/configs/experiment/cb55_full_run_unet.yaml +++ b/configs/experiment/cb55_full_run_unet.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: experiment.yaml - /plugins: default.yaml - /task: semantic_segmentation_HisDB_cropped.yaml - /loss: crossentropyloss.yaml @@ -23,6 +24,8 @@ defaults: # instead we define all modules and their paths directly in this config, # so everything is stored in one place for more readibility +name: "semantic_segmentation_cb55_full_unet" + train: True test: True predict: False @@ -62,6 +65,6 @@ callbacks: logger: wandb: - name: 'cb55-full-unet' + name: ${name} tags: ["best_model", "USL", "baseline"] group: 'baseline' diff --git a/configs/experiment/cb55_select_train15_unet.yaml b/configs/experiment/cb55_select_train15_unet.yaml index 7f177d56..ee0e593a 100644 --- a/configs/experiment/cb55_select_train15_unet.yaml +++ b/configs/experiment/cb55_select_train15_unet.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: experiment.yaml - /plugins: default.yaml - /task: semantic_segmentation_HisDB_cropped.yaml - /loss: crossentropyloss.yaml @@ -23,6 +24,8 @@ defaults: # instead we define all modules and their paths directly in this config, # so everything is stored in one place for more readibility +name: "semantic_segmentation_cb55_train15_unet" + train: True test: True predict: False @@ -63,6 +66,6 @@ callbacks: logger: wandb: - name: 'cb55-select-train15-unet' + name: ${name} tags: ["best_model", "USL", "baseline"] group: 'baseline' diff --git a/configs/experiment/cb55_select_train1_val1_unet.yaml b/configs/experiment/cb55_select_train1_val1_unet.yaml index 6eb5a59c..b89de9dc 100644 --- a/configs/experiment/cb55_select_train1_val1_unet.yaml +++ b/configs/experiment/cb55_select_train1_val1_unet.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: experiment.yaml - /plugins: default.yaml - /task: semantic_segmentation_HisDB_cropped.yaml - /loss: crossentropyloss.yaml @@ -23,6 +24,8 @@ defaults: # instead we define all modules and their paths directly in this config, # so everything is stored in one place for more readibility +name: "semantic_segmentation_cb55_train1_val1_unet" + train: True test: True predict: False @@ -64,6 +67,6 @@ callbacks: logger: wandb: - name: 'cb55-select-train1-val1-unet' + name: ${name} tags: ["best_model", "USL", "baseline"] group: 'baseline' diff --git a/configs/experiment/dev_rgb_full_page.yaml b/configs/experiment/dev_rgb_full_page.yaml index 3a040a23..7560165f 100644 --- a/configs/experiment/dev_rgb_full_page.yaml +++ b/configs/experiment/dev_rgb_full_page.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: development.yaml - /plugins: default.yaml - /task: semantic_segmentation_RGB.yaml - /loss: crossentropyloss.yaml diff --git a/configs/experiment/dev_rgb_full_page_predict.yaml b/configs/experiment/dev_rgb_full_page_predict.yaml index d3be87de..9f0466f1 100644 --- a/configs/experiment/dev_rgb_full_page_predict.yaml +++ b/configs/experiment/dev_rgb_full_page_predict.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: development.yaml - /plugins: default.yaml - /task: semantic_segmentation_RGB.yaml - /loss: crossentropyloss.yaml diff --git a/configs/experiment/dev_rolf_format.yaml b/configs/experiment/dev_rolf_format.yaml index 25ee04b2..b3193f82 100644 --- a/configs/experiment/dev_rolf_format.yaml +++ b/configs/experiment/dev_rolf_format.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: development.yaml - /plugins: default.yaml - /task: semantic_segmentation_RGB.yaml - /datamodule: rolf_format_dev.yaml diff --git a/configs/experiment/dev_rolf_format_prediction.yaml b/configs/experiment/dev_rolf_format_prediction.yaml index 1e36e7dc..4256868b 100644 --- a/configs/experiment/dev_rolf_format_prediction.yaml +++ b/configs/experiment/dev_rolf_format_prediction.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: development.yaml - /plugins: default.yaml - /task: semantic_segmentation_RGB.yaml - /datamodule: rolf_format_dev_prediction.yaml diff --git a/configs/experiment/dev_rotnet_cnn_basic_cb55_10.yaml b/configs/experiment/dev_rotnet_cnn_basic_cb55_10.yaml index af715dea..154ad465 100644 --- a/configs/experiment/dev_rotnet_cnn_basic_cb55_10.yaml +++ b/configs/experiment/dev_rotnet_cnn_basic_cb55_10.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: development.yaml - /plugins: default.yaml - /task: classification.yaml - /loss: crossentropyloss.yaml diff --git a/configs/experiment/dev_rotnet_pt_resnet18_cb55_10_segmentation.yaml b/configs/experiment/dev_rotnet_pt_resnet18_cb55_10_segmentation.yaml index 57e4eabe..a4c0570b 100644 --- a/configs/experiment/dev_rotnet_pt_resnet18_cb55_10_segmentation.yaml +++ b/configs/experiment/dev_rotnet_pt_resnet18_cb55_10_segmentation.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: development.yaml - /plugins: default.yaml - /task: semantic_segmentation_HisDB_cropped.yaml - /loss: crossentropyloss.yaml diff --git a/configs/experiment/dev_rotnet_resnet18_cb55_10.yaml b/configs/experiment/dev_rotnet_resnet18_cb55_10.yaml index 559c37fe..5c71fab6 100644 --- a/configs/experiment/dev_rotnet_resnet18_cb55_10.yaml +++ b/configs/experiment/dev_rotnet_resnet18_cb55_10.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: development.yaml - /plugins: default.yaml - /task: classification.yaml - /loss: crossentropyloss.yaml diff --git a/configs/experiment/dev_rotnet_resnet50_cb55_10.yaml b/configs/experiment/dev_rotnet_resnet50_cb55_10.yaml index f82d5895..60a0095a 100644 --- a/configs/experiment/dev_rotnet_resnet50_cb55_10.yaml +++ b/configs/experiment/dev_rotnet_resnet50_cb55_10.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: development.yaml - /plugins: default.yaml - /task: classification.yaml - /loss: crossentropyloss.yaml diff --git a/configs/experiment/development_baby_unet_cb55_10.yaml b/configs/experiment/development_baby_unet_cb55_10.yaml index 9604d72c..e748dedd 100644 --- a/configs/experiment/development_baby_unet_cb55_10.yaml +++ b/configs/experiment/development_baby_unet_cb55_10.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: development.yaml - /plugins: default.yaml - /task: semantic_segmentation_HisDB_cropped.yaml - /loss: crossentropyloss.yaml diff --git a/configs/experiment/development_baby_unet_rgb_data.yaml b/configs/experiment/development_baby_unet_rgb_data.yaml index d2f113c4..9de81b39 100644 --- a/configs/experiment/development_baby_unet_rgb_data.yaml +++ b/configs/experiment/development_baby_unet_rgb_data.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: development.yaml - /plugins: default.yaml - /task: semantic_segmentation_RGB_cropped.yaml - /loss: crossentropyloss.yaml diff --git a/configs/experiment/rotnet_resnet18_cb55_full.yaml b/configs/experiment/rotnet_resnet18_cb55_full.yaml index 9722d8fd..f951bc54 100644 --- a/configs/experiment/rotnet_resnet18_cb55_full.yaml +++ b/configs/experiment/rotnet_resnet18_cb55_full.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: experiment.yaml - /plugins: default.yaml - /task: classification.yaml - /loss: crossentropyloss.yaml @@ -24,6 +25,8 @@ defaults: # instead we define all modules and their paths directly in this config, # so everything is stored in one place for more readibility +name: "rotnet_cb55_full_resnet18" + train: True test: False predict: False @@ -67,6 +70,6 @@ callbacks: logger: wandb: - name: 'rotnet-resnet18-cb55' + name: ${name} tags: [ "best_model", "USL"] group: 'rotnet-baseline' diff --git a/configs/experiment/rotnet_resnet18_cb55_train10_last.yaml b/configs/experiment/rotnet_resnet18_cb55_train10_last.yaml index d89da333..02e5a93b 100644 --- a/configs/experiment/rotnet_resnet18_cb55_train10_last.yaml +++ b/configs/experiment/rotnet_resnet18_cb55_train10_last.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: experiment.yaml - /plugins: default.yaml - /task: classification.yaml - /loss: crossentropyloss.yaml @@ -24,6 +25,8 @@ defaults: # instead we define all modules and their paths directly in this config, # so everything is stored in one place for more readibility +name: "rotnet_cb55_train10_resnet18" + train: True test: False predict: False @@ -78,6 +81,6 @@ callbacks: logger: wandb: - name: 'rotnet-resnet18-cb55-last10-train' + name: ${name} tags: [ "best_model", "USL"] group: 'rotnet-baseline' diff --git a/configs/experiment/rotnet_resnet18_cb55_train19_last.yaml b/configs/experiment/rotnet_resnet18_cb55_train19_last.yaml index aaf13797..54574ef1 100644 --- a/configs/experiment/rotnet_resnet18_cb55_train19_last.yaml +++ b/configs/experiment/rotnet_resnet18_cb55_train19_last.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: experiment.yaml - /plugins: default.yaml - /task: classification.yaml - /loss: crossentropyloss.yaml @@ -24,6 +25,8 @@ defaults: # instead we define all modules and their paths directly in this config, # so everything is stored in one place for more readibility +name: "rotnet_cb55_train19_resnet18" + train: True test: False predict: False @@ -87,6 +90,6 @@ callbacks: logger: wandb: - name: 'rotnet-resnet18-cb55-last19-train' + name: ${name} tags: [ "best_model", "USL"] group: 'rotnet-baseline' diff --git a/configs/experiment/synthetic_baby_unet_layoutD_gtD.yaml b/configs/experiment/synthetic_baby_unet_layoutD_gtD.yaml index 3f8d69fd..96a136fc 100644 --- a/configs/experiment/synthetic_baby_unet_layoutD_gtD.yaml +++ b/configs/experiment/synthetic_baby_unet_layoutD_gtD.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: experiment.yaml - /plugins: default.yaml - /task: semantic_segmentation_RGB_cropped.yaml - /loss: crossentropyloss.yaml @@ -24,6 +25,8 @@ defaults: # instead we define all modules and their paths directly in this config, # so everything is stored in one place for more readibility +name: "semantic_segmentation_synthetic_layoutD_gtD_baby_unet" + train: True test: True predict: False @@ -64,6 +67,6 @@ callbacks: logger: wandb: - name: 'synthetic-baby-unet-layoutD-gtD' + name: ${name} tags: [ "best_model", "synthetic", "layoutD", "gtD", "Rolf" ] group: 'synthetic' diff --git a/configs/experiment/synthetic_baby_unet_layoutD_gtD_rolf_format.yaml b/configs/experiment/synthetic_baby_unet_layoutD_gtD_rolf_format.yaml index 3acade42..078c44a2 100644 --- a/configs/experiment/synthetic_baby_unet_layoutD_gtD_rolf_format.yaml +++ b/configs/experiment/synthetic_baby_unet_layoutD_gtD_rolf_format.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: experiment.yaml - /plugins: default.yaml - /task: semantic_segmentation_RGB.yaml - /datamodule: rolf_format_layoutD_gtD.yaml @@ -25,6 +26,8 @@ defaults: # instead we define all modules and their paths directly in this config, # so everything is stored in one place for more readibility +name: "semantic_segmentation_synthetic_fullpage_layoutD_gtD_baby_unet" + train: True test: True predict: False @@ -53,6 +56,6 @@ callbacks: logger: wandb: - name: 'synthetic-baby-unet-layoutD-gtD-rolf-format' + name: ${name} tags: [ "best_model", "synthetic", "RolfFormat" ] group: 'synthetic' diff --git a/configs/experiment/synthetic_baby_unet_layoutR_gtD.yaml b/configs/experiment/synthetic_baby_unet_layoutR_gtD.yaml index d8ebb215..fe59cbe5 100644 --- a/configs/experiment/synthetic_baby_unet_layoutR_gtD.yaml +++ b/configs/experiment/synthetic_baby_unet_layoutR_gtD.yaml @@ -4,6 +4,7 @@ # python run.py +experiment=exp_example_full defaults: + - /mode: experiment.yaml - /plugins: default.yaml - /task: semantic_segmentation_RGB_cropped.yaml - /loss: crossentropyloss.yaml @@ -24,6 +25,8 @@ defaults: # instead we define all modules and their paths directly in this config, # so everything is stored in one place for more readibility +name: "semantic_segmentation_synthetic_layoutR_gtD_baby_unet" + train: True test: True predict: False @@ -64,6 +67,6 @@ callbacks: logger: wandb: - name: 'synthetic-baby-unet-layoutR-gtD' + name: ${name} tags: [ "best_model", "synthetic", "layoutR", "gtD", "Rolf" ] group: 'synthetic' diff --git a/configs/hydra/default.yaml b/configs/hydra/default.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/configs/mode/development.yaml b/configs/mode/development.yaml new file mode 100644 index 00000000..7d3a60b5 --- /dev/null +++ b/configs/mode/development.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +experiment_mode: True + +# name of the experiment +name: default + +hydra: + run: + dir: outputs/development/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file diff --git a/configs/mode/experiment.yaml b/configs/mode/experiment.yaml new file mode 100644 index 00000000..ce71819d --- /dev/null +++ b/configs/mode/experiment.yaml @@ -0,0 +1,10 @@ +# @package _global_ + +experiment_mode: True + +# name of the experiment +name: ??? + +hydra: + run: + dir: outputs/experiments/${name}/${now:%Y-%m-%d}/${now:%H-%M-%S} \ No newline at end of file diff --git a/src/utils/utils.py b/src/utils/utils.py index 44a486f2..1d64f3d3 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -1,5 +1,6 @@ import logging import random +import sys import warnings from typing import List, Sequence @@ -86,6 +87,10 @@ def check_config(config: DictConfig) -> None: if config.trainer.get("accelerator") == 'ddp_cpu' and config.trainer.precision == 16: log.warning(f'You are using ddp_cpu without precision=16. This can lead to a crash! Use 64 or 32!') + if config.get('experiment_mode') and not config.get('name'): + log.info("Experiment mode without specifying a name!") + sys.exit(1) + # Set seed for random number generators in pytorch, numpy and python.random if "seed" in config: seed_everything(config.seed, workers=True) From d5b8f14698e5e8c05c8f5e9e160c2044af77aeb6 Mon Sep 17 00:00:00 2001 From: Lars Voegtlin Date: Mon, 29 Nov 2021 15:06:33 +0100 Subject: [PATCH 2/4] :wrench: update development mode --- configs/mode/development.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/configs/mode/development.yaml b/configs/mode/development.yaml index 7d3a60b5..952a3fc7 100644 --- a/configs/mode/development.yaml +++ b/configs/mode/development.yaml @@ -1,9 +1,6 @@ # @package _global_ -experiment_mode: True - -# name of the experiment -name: default +development_mode: True hydra: run: From ca74c10074c737ec70913f05cba75dbdc52d3310 Mon Sep 17 00:00:00 2001 From: Lars Voegtlin Date: Mon, 29 Nov 2021 18:05:34 +0100 Subject: [PATCH 3/4] :wrench: make less val epochs --- configs/experiment/cb55_full_run_unet.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/experiment/cb55_full_run_unet.yaml b/configs/experiment/cb55_full_run_unet.yaml index 0e4377ae..3ac827aa 100644 --- a/configs/experiment/cb55_full_run_unet.yaml +++ b/configs/experiment/cb55_full_run_unet.yaml @@ -37,6 +37,7 @@ trainer: min_epochs: 1 max_epochs: 50 precision: 16 + check_val_every_n_epoch: 4 task: confusion_matrix_log_every_n_epoch: 10 From 164198de565a07725bf0c1eb67d68855bdcd85e7 Mon Sep 17 00:00:00 2001 From: Lars Voegtlin Date: Mon, 29 Nov 2021 23:50:59 +0100 Subject: [PATCH 4/4] :wrench: no weight loss cb55 unet --- .../cb55_full_run_unet_loss_no_weights.yaml | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 configs/experiment/cb55_full_run_unet_loss_no_weights.yaml diff --git a/configs/experiment/cb55_full_run_unet_loss_no_weights.yaml b/configs/experiment/cb55_full_run_unet_loss_no_weights.yaml new file mode 100644 index 00000000..1f41ff34 --- /dev/null +++ b/configs/experiment/cb55_full_run_unet_loss_no_weights.yaml @@ -0,0 +1,71 @@ +# @package _global_ + +# to execute this experiment run: +# python run.py +experiment=exp_example_full + +defaults: + - /mode: experiment.yaml + - /plugins: default.yaml + - /task: semantic_segmentation_HisDB_cropped.yaml + - /loss: crossentropyloss_no_weight.yaml + - /metric: hisdbiou.yaml + - /model/backbone: unet_model.yaml + - /model/header: identity.yaml + - /optimizer: adam.yaml + - /callbacks: + - check_compatibility.yaml + - model_checkpoint.yaml + - /logger: + - wandb.yaml # set logger here or use command line (e.g. `python run.py logger=wandb`) + - csv.yaml + - _self_ + +# we override default configurations with nulls to prevent them from loading at all +# instead we define all modules and their paths directly in this config, +# so everything is stored in one place for more readibility + +name: "semantic_segmentation_cb55_full_unet_loss_no_weights" + +train: True +test: True +predict: False + +trainer: + _target_: pytorch_lightning.Trainer + gpus: -1 + accelerator: 'ddp' + min_epochs: 1 + max_epochs: 50 + precision: 16 + check_val_every_n_epoch: 4 + +task: + confusion_matrix_log_every_n_epoch: 10 + confusion_matrix_val: True + confusion_matrix_test: True + +datamodule: + _target_: src.datamodules.DivaHisDB.datamodule_cropped.DivaHisDBDataModuleCropped + + data_dir: /net/research-hisdoc/datasets/semantic_segmentation/datasets_cropped/CB55 + crop_size: 256 + num_workers: 4 + batch_size: 16 + shuffle: True + drop_last: True + data_folder_name: data + gt_folder_name: gt + +callbacks: + model_checkpoint: + monitor: "val/hisdbiou" + mode: "max" + filename: ${checkpoint_folder_name}cb55-full-unet +# watch_model: +# log_freq: 1000 + +logger: + wandb: + name: ${name} + tags: ["best_model", "USL", "baseline"] + group: 'baseline'