Merge pull request #86 from DIVA-DIA/dev

release 0.2.0
DIVA-DIA · Nov 25, 2021 · 931a4bd · 931a4bd
2 parents f926959 + 660dc46
commit 931a4bd
Show file tree

Hide file tree

Showing 146 changed files with 6,858 additions and 1,082 deletions.
diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml
@@ -63,13 +63,13 @@ jobs:
     - name: Statistics
       if: success()
       run: |
-        coverage report -m
+        coverage report -m --omit="*/test*"
 
     - name: Produce statistics html
       if: success()
       run: |
-        coverage html
-        coverage xml
+        coverage html --omit="*/test*"
+        coverage xml --omit="*/test*"
 
     - name: Code coverage results upload
       if: success()

diff --git a/README.md b/README.md
@@ -6,6 +6,7 @@
 <a href="https://pytorchlightning.ai/"><img alt="Lightning" src="https://img.shields.io/badge/-Lightning-792ee5"></a>
 <a href="https://hydra.cc/"><img alt="Config: Hydra" src="https://img.shields.io/badge/Config-Hydra-89b8cd"></a>
 <a href="https://github.com/ashleve/lightning-hydra-template"><img alt="Template" src="https://img.shields.io/badge/-Lightning--Hydra--Template-017F2F?style=flat&logo=github&labelColor=gray"></a><br>
+![tests](https://github.com/DIVA-DIA/unsupervised_learning/actions/workflows/ci-testing.yml/badge.svg)
 
 [comment]: <> ([![Paper]&#40;http://img.shields.io/badge/paper-arxiv.1001.2234-B31B1B.svg&#41;]&#40;https://www.nature.com/articles/nature14539&#41;)
 
@@ -94,3 +95,35 @@ python run.py trainer.max_epochs=20 datamodule.batch_size=64
 11. Go to the root folder of the framework and activate the environment (source .autoenv OR conda activate unsupervised_learning)
 12. Log into wandb. Execute `wandb login` and follow the instructions
 13. Now you should be able to run the basic experiment from PyCharm
+
+
+### Loading models
+You can load the different model parts `backbone` or `header` as well as the whole task.
+To load the `backbone` or the `header` you need to add to your experiment config the field `path_to_weights`.
+e.g.
+```
+model:
+    header:
+        path_to_weights: /my/path/to/the/pth/file
+```
+To load the whole task you need to provide the path to the whole task to the trainer. This is with the field `resume_from_checkpoint`.
+e.g.
+```
+trainer:
+    resume_from_checkpoint: /path/to/.ckpt/file
+```
+
+### Freezing model parts
+You can freeze both parts of the model (backbone or header) with the `freeze` flag in the config. 
+E.g. you want to freeze the backbone:
+In the command line:
+```
+python run.py +model.backbone.freeze=True
+```
+In the config (e.g. model/backbone/baby_unet_model.yaml):
+```
+...
+freeze: True
+...
+```
+CARE: You can not train a model when you do not have trainable parameters (e.g. freezing backbone and header).
diff --git a/configs/datamodule/cb55_10_cropped_datamodule.yaml b/configs/datamodule/cb55_10_cropped_datamodule.yaml
@@ -1,7 +1,7 @@
-_target_: src.datamodules.hisDBDataModule.DIVAHisDBDataModule.DIVAHisDBDataModuleCropped
+_target_: src.datamodules.DivaHisDB.datamodule_cropped.DivaHisDBDataModuleCropped
 
 
-data_dir: /netscratch/datasets/semantic_segmentation/datasets_cropped/CB55-10-segmentation
+data_dir: /net/research-hisdoc/datasets/semantic_segmentation/datasets_cropped/CB55-10-segmentation
 crop_size: 256
 num_workers: 4
 batch_size: 16

diff --git a/configs/datamodule/cb55_cropped_datamodule.yaml b/configs/datamodule/cb55_cropped_datamodule.yaml
@@ -1,6 +1,6 @@
-_target_: src.datamodules.hisDBDataModule.DIVAHisDBDataModule.DIVAHisDBDataModuleCropped
+_target_: src.datamodules.DivaHisDB.datamodule_cropped.DivaHisDBDataModuleCropped
 
-data_dir: /netscratch/datasets/semantic_segmentation/datasets_cropped/CB55
+data_dir: /net/research-hisdoc/datasets/semantic_segmentation/datasets_cropped/CB55
 crop_size: 256
 num_workers: 4
 batch_size: 16

diff --git a/configs/datamodule/rolf_format_dev.yaml b/configs/datamodule/rolf_format_dev.yaml
@@ -0,0 +1,164 @@
+_target_: src.datamodules.RolfFormat.datamodule.DataModuleRolfFormat
+
+num_workers: 4
+batch_size: 8
+shuffle: True
+drop_last: False
+
+data_root: /net/research-hisdoc/datasets/semantic_segmentation/rolf_format
+
+train_specs:
+    append1:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutD/data"
+        doc_names:  "A1-MD-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutD/gtruth"
+        gt_names:   "A1-MD-truthD-####.gif"
+        range_from: 1000
+        range_to:   1059
+    append2:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutR/data"
+        doc_names:  "A1-MR-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutR/gtruth"
+        gt_names:   "A1-MR-truthD-####.gif"
+        range_from: 1000
+        range_to:   1059
+
+val_specs:
+    append1:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutD/data"
+        doc_names:  "A1-MD-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutD/gtruth"
+        gt_names:   "A1-MD-truthD-####.gif"
+        range_from: 1060
+        range_to:   1079
+    append2:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutR/data"
+        doc_names:  "A1-MR-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutR/gtruth"
+        gt_names:   "A1-MR-truthD-####.gif"
+        range_from: 1060
+        range_to:   1079
+
+test_specs:
+    append1:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutD/data"
+        doc_names:  "A1-MD-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutD/gtruth"
+        gt_names:   "A1-MD-truthD-####.gif"
+        range_from: 1080
+        range_to:   1099
+    append2:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutR/data"
+        doc_names:  "A1-MR-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutR/gtruth"
+        gt_names:   "A1-MR-truthD-####.gif"
+        range_from: 1080
+        range_to:   1099
+    append3:
+        doc_dir:    "../synthetic/SetA1_sizeM/layoutR/split/test/data"
+        doc_names:  "A1-MR-page-####.jpg"
+        gt_dir:     "../synthetic/SetA1_sizeM/layoutR/split/test/gtD"
+        gt_names:   "A1-MR-page-####.gif"
+        range_from: 1080
+        range_to:   1099
+
+
+image_dims:
+    width:  640
+    height: 896
+
+image_analytics:
+    mean:
+        R: 0.8664800196201524
+        G: 0.7408864118075618
+        B: 0.6299955083595935
+    std:
+        R: 0.2156624188591712
+        G: 0.20890185198454636
+        B: 0.1870731300038113
+
+classes:
+    class0:
+        color:
+            R: 0
+            G: 0
+            B: 0
+        weight: 0.0016602289391364547
+    class1:
+        color:
+            R: 0
+            G: 102
+            B: 0
+        weight: 0.22360020547468618
+    class2:
+        color:
+            R: 0
+            G: 102
+            B: 102
+        weight: 0.014794833923108578
+    class3:
+        color:
+            R: 0
+            G: 153
+            B: 153
+        weight: 0.05384506923533185
+    class4:
+        color:
+            R: 0
+            G: 255
+            B: 0
+        weight: 0.1115978481679602
+    class5:
+        color:
+            R: 0
+            G: 255
+            B: 255
+        weight: 0.037436533973406926
+    class6:
+        color:
+            R: 102
+            G: 0
+            B: 0
+        weight: 0.12569866772812885
+    class7:
+        color:
+            R: 102
+            G: 0
+            B: 102
+        weight: 0.03591164457353043
+    class8:
+        color:
+            R: 102
+            G: 102
+            B: 0
+        weight: 0.01062086078798502
+    class9:
+        color:
+            R: 153
+            G: 0
+            B: 153
+        weight: 0.1491578366712268
+    class10:
+        color:
+            R: 153
+            G: 153
+            B: 0
+        weight: 0.0414074692141804
+    class11:
+        color:
+            R: 255
+            G: 0
+            B: 0
+        weight: 0.08600602291055298
+    class12:
+        color:
+            R: 255
+            G: 0
+            B: 255
+        weight: 0.08349157426652898
+    class13:
+        color:
+            R: 255
+            G: 255
+            B: 0
+        weight: 0.024771204134236315
diff --git a/configs/datamodule/rolf_format_dev_no_weights.yaml b/configs/datamodule/rolf_format_dev_no_weights.yaml
@@ -0,0 +1,142 @@
+_target_: src.datamodules.RolfFormat.datamodule.DataModuleRolfFormat
+
+num_workers: 4
+batch_size: 8
+shuffle: True
+drop_last: False
+
+data_root: /net/research-hisdoc/datasets/semantic_segmentation/rolf_format
+
+train_specs:
+    append1:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutD/data"
+        doc_names:  "A1-MD-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutD/gtruth"
+        gt_names:   "A1-MD-truthD-####.gif"
+        range_from: 1000
+        range_to:   1059
+    append2:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutR/data"
+        doc_names:  "A1-MR-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutR/gtruth"
+        gt_names:   "A1-MR-truthD-####.gif"
+        range_from: 1000
+        range_to:   1059
+
+val_specs:
+    append1:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutD/data"
+        doc_names:  "A1-MD-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutD/gtruth"
+        gt_names:   "A1-MD-truthD-####.gif"
+        range_from: 1060
+        range_to:   1079
+    append2:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutR/data"
+        doc_names:  "A1-MR-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutR/gtruth"
+        gt_names:   "A1-MR-truthD-####.gif"
+        range_from: 1060
+        range_to:   1079
+
+test_specs:
+    append1:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutD/data"
+        doc_names:  "A1-MD-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutD/gtruth"
+        gt_names:   "A1-MD-truthD-####.gif"
+        range_from: 1080
+        range_to:   1099
+    append2:
+        doc_dir:    "SetA1_sizeM_Rolf/layoutR/data"
+        doc_names:  "A1-MR-page-####.jpg"
+        gt_dir:     "SetA1_sizeM_Rolf/layoutR/gtruth"
+        gt_names:   "A1-MR-truthD-####.gif"
+        range_from: 1080
+        range_to:   1099
+
+image_dims:
+    width:  640
+    height: 896
+
+image_analytics:
+    mean:
+        R: 0.8664800196201524
+        G: 0.7408864118075618
+        B: 0.6299955083595935
+    std:
+        R: 0.2156624188591712
+        G: 0.20890185198454636
+        B: 0.1870731300038113
+
+classes:
+    class0:
+        color:
+            R: 0
+            G: 0
+            B: 0
+    class1:
+        color:
+            R: 0
+            G: 102
+            B: 0
+    class2:
+        color:
+            R: 0
+            G: 102
+            B: 102
+    class3:
+        color:
+            R: 0
+            G: 153
+            B: 153
+    class4:
+        color:
+            R: 0
+            G: 255
+            B: 0
+    class5:
+        color:
+            R: 0
+            G: 255
+            B: 255
+    class6:
+        color:
+            R: 102
+            G: 0
+            B: 0
+    class7:
+        color:
+            R: 102
+            G: 0
+            B: 102
+    class8:
+        color:
+            R: 102
+            G: 102
+            B: 0
+    class9:
+        color:
+            R: 153
+            G: 0
+            B: 153
+    class10:
+        color:
+            R: 153
+            G: 153
+            B: 0
+    class11:
+        color:
+            R: 255
+            G: 0
+            B: 0
+    class12:
+        color:
+            R: 255
+            G: 0
+            B: 255
+    class13:
+        color:
+            R: 255
+            G: 255
+            B: 0