synchronize on GPU devices for CUDA-enabled processors, fix #1:

- make must not run tools that use GPU resources in parallel
  (according to max-jobs/load-level) like the others, because
  they will race against each other for GPU memory; to still
  allow CPU-parallel execution with such tools in a workflow,
  introduce a shell semaphore (counting GPUs) for all tools
  that have the (target-specific) variable `GPU` set, and
  fallback to CPU if no GPU is available within 3 seconds
- all OCR rules involving Calamari: set GPU = 1
- tesserocr-deskew: make default confidence threshold explicit
- update README

Makefile

@@ -169,13 +169,36 @@ else
 # FIXME: Also, this does not cover multiple output filegrps
 # (ignoring them will give side effects!)
 TOOL =
-PARAMS = 
+GPU =
+PARAMS =
+
+define toolrecipe =
+$(TOOL) -I $< -O $@ -p $@.json 2>&1 | tee $@.log && \
+	touch $@ || { \
+	rm -fr $@.json $@; exit 1; }
+endef
+# Extra recipe to control allocation of GPU resources
+# (for processors explicitly configured as CUDA-enabled):
+# If not enough GPUs are available for a new processor
+# at any given time, then invocation should fallback to CPU.
+# (This requires CUDA toolkit and GNU parallel.)
+gputoolrecipe = $(toolrecipe)
+ifneq ($(shell which nvidia-smi),)
+ifneq ($(shell which sem),)
+NGPUS = $(shell nvidia-smi -L | wc -l)
+define gputoolrecipe =
+if sem --id OCR-D-GPUSEM -j $(NGPUS) --st -3 true 2>/dev/null; then \
+   sem --id OCR-D-GPUSEM -j $(NGPUS) --fg $(toolrecipe); else \
+   CUDA_VISIBLE_DEVICES= $(toolrecipe); fi
+endef
+else
+$(warning You risk running into GPU races. Install GNU parallel to synchronize CUDA-enabled processors.)
+endif
+endif
 %:
 	-ocrd workspace remove-group -r $@ 2>/dev/null
 	$(file > $@.json, { $(PARAMS) })
-	$(TOOL) -I $< -O $@ -p $@.json 2>&1 | tee $@.log && \
-		touch $@ || { \
-		rm -fr $@.json $@; exit 1; }
+	$(if $(GPU),$(gputoolrecipe),$(toolrecipe))
 
 view:
 # filter out file groups we do not need for current configuration:

README.md

@@ -58,7 +58,12 @@ Next, edit the file to your needs: Write rules using file groups as prerequisite
 - Change/customize at least the `info` target, and the `INPUT` and `OUTPUT` name/rule.
 - Copy/paste rules from the existing configurations.
 - Define variables with the names of all target/prerequisite file groups, so rules and dependent targets can re-use them (and the names can be easily changed later).
-- Try to use the static pattern rule (which takes the target as output file group and the prerequisite as input file group) for most OCR-D CLIs – unless you have more than 1 input or output file group, or your operation does not use an OCR-D CLI – by simply defining the target-specific variable `TOOL` (and optionally `PARAMS`) and giving no recipe whatsoever .
+- Try to use the static pattern rule (which takes the target as output file group and the prerequisite as input file group) for most OCR-D CLIs – unless you have more than 1 input or output file group, or your operation does not use an OCR-D CLI – by simply defining the target-specific variable `TOOL` (and optionally `PARAMS`) and giving no recipe whatsoever.
+- When your processor uses GPU resources, you must prevent races for GPU memory during parallel execution.
+
+  You can achieve this by simply setting `GPU = 1` when using the static pattern rule, or by using `sem --id OCR-D-GPUSEM` in your own recipes.
+
+  Alternatively, you can either prevent using GPUs globally by (un)setting `CUDA_VISIBLE_DEVICES=`, or using multiple CPUs by not running with `-j`.
 
 #### Example
 
@@ -89,7 +94,7 @@ OUTPUT = EVAL
 # more than 1 input file group and no output file group:
 $(OUTPUT): $(OCR)
 $(OUTPUT): $(INPUT)
-	ocrd-cor-asv-ann-evaluate -I `echo $^ | tr ' ' ,`
+	ocrd-cor-asv-ann-evaluate -I $(call concatcomma,$^)
 
 # Because no file group (directory) is created,
 # we have to declare this target as phony:
@@ -99,6 +104,9 @@ $(OUTPUT): $(INPUT)
 # we must override the default goal to be our desired target:
 .DEFAULT_GOAL = $(OUTPUT)
 
+comma = ,
+concatcomma = $(subst $() $(),$(comma),$(1))
+
 # Always necessary:
 include Makefile
 ```
@@ -107,7 +115,7 @@ include Makefile
 
 #### OCR-D ground truth
 
-For the `bagit_data_text_structur` repository, which includes both layout and text annotation down to the textline level, but very coarse segmentation, the following _character error rate_ (CER) was measured:
+For the `data_structure_text/dta` repository, which includes both layout and text annotation down to the textline level, but very coarse segmentation, the following _character error rate_ (CER) was measured:
 
 | *pipeline configuration* | *CER* |
 | ---------- | ----- |
@@ -208,3 +216,11 @@ Rules that are not configuration-specific (like the static pattern rule) are all
   targets are the configured file groups, including the local default goal.
 
 The former calls the latter recursively for each workspace.
+
+#### GPU vs CPU parallelism
+
+When executing workflows in parallel (with `--jobs`) on multiple CPUs, it must be ensured that not too many processors are running at any time which use GPU resources. Thus, make needs to know:
+1. which processors (have/want to) share GPU resources, and
+2. how many such processors can run in parallel.
+
+It can then synchronize these processors with a semaphore. This is achieved by expanding the static pattern rule with a synchronisation mechanism (based on GNU parallel). Workflow configurations can use that by setting the target-specific variable `GPU` to a non-empty value for the respective rules. (Custom recipes will have to use `sem --id OCR-D-GPUSEM`.)

gt-binarize-page-olena-sauvola-clip-resegment-dewarp-ocr-ocropy-tesseract.mk

@@ -67,6 +67,7 @@ $(OCR6): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model"
 $(OCR7): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "GT4HistOCR_2000000+GT4HistOCR_300000+GT4HistOCR_100000"
 
 $(OCR8): TOOL = ocrd-calamari-recognize
+$(OCR8): GPU = 1
 $(OCR8): PARAMS = "checkpoint" : "$(VIRTUAL_ENV)/share/calamari/GT4HistOCR/*.ckpt.json"
 
 OUTPUT = $(DEW)-OCR

gt-binarize-page-olena-sauvola-denoise-ocropy-clip-resegment-dewarp-ocr-ocropy-tesseract.mk

@@ -73,6 +73,7 @@ $(OCR6): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model"
 $(OCR7): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "GT4HistOCR_2000000+GT4HistOCR_300000+GT4HistOCR_100000"
 
 $(OCR8): TOOL = ocrd-calamari-recognize
+$(OCR8): GPU = 1
 $(OCR8): PARAMS = "checkpoint" : "$(VIRTUAL_ENV)/share/calamari/GT4HistOCR/*.ckpt.json"
 
 OUTPUT = $(DEW)-OCR

gt-binarize-page-olena-sauvola-denoise-ocropy-deskew-page-ocropy-clip-deskew-region-tesseract-resegment-dewarp-ocr-ocropy-tesseract-extract-lines.mk

@@ -66,23 +66,30 @@ $(DEW): TOOL = ocrd-cis-ocropy-dewarp
 OCR1 = $(DEW:$(INPUT)-%=OCR-D-OCR-OCRO-fraktur-%)
 OCR2 = $(DEW:$(INPUT)-%=OCR-D-OCR-OCRO-frakturjze-%)
 OCR3 = $(DEW:$(INPUT)-%=OCR-D-OCR-TESS-Fraktur-%)
-OCR4 = $(DEW:$(INPUT)-%=OCR-D-OCR-TESS-frk-%)
-OCR5 = $(DEW:$(INPUT)-%=OCR-D-OCR-TESS-frk+deu-%)
-OCR6 = $(DEW:$(INPUT)-%=OCR-D-OCR-TESS-gt4histocr-%)
+OCR4 = $(DEW:$(INPUT)-%=OCR-D-OCR-TESS-Fraktur+Latin-%)
+OCR5 = $(DEW:$(INPUT)-%=OCR-D-OCR-TESS-frk-%)
+OCR6 = $(DEW:$(INPUT)-%=OCR-D-OCR-TESS-frk+deu-%)
+OCR7 = $(DEW:$(INPUT)-%=OCR-D-OCR-TESS-gt4histocr-%)
+OCR8 = $(DEW:$(INPUT)-%=OCR-D-OCR-CALA-gt4histocr-%)
 
-$(OCR1) $(OCR2) $(OCR3) $(OCR4) $(OCR5) $(OCR6): $(DEW)
+$(OCR1) $(OCR2) $(OCR3) $(OCR4) $(OCR5) $(OCR6) $(OCR7) $(OCR8): $(DEW)
 
 $(OCR1) $(OCR2): TOOL = ocrd-cis-ocropy-recognize
 $(OCR1): PARAMS = "textequiv_level": "glyph", "model": "fraktur.pyrnn"
 $(OCR2): PARAMS = "textequiv_level": "glyph", "model": "fraktur-jze.pyrnn"
 
-$(OCR3) $(OCR4) $(OCR5) $(OCR6): TOOL = ocrd-tesserocr-recognize
+$(OCR3) $(OCR4) $(OCR5) $(OCR6) $(OCR7): TOOL = ocrd-tesserocr-recognize
 $(OCR3): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "script/Fraktur"
-$(OCR4): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "frk"
-$(OCR5): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "frk+deu"
-$(OCR6): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "GT4HistOCR_2000000"
+$(OCR4): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "script/Fraktur+script/Latin"
+$(OCR5): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "frk"
+$(OCR6): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "frk+deu"
+$(OCR7): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "GT4HistOCR_2000000+GT4HistOCR_300000+GT4HistOCR_100000"
 
-LINES = $(patsubst %,OCR-D-IMG-LINES-%,$(DEW) $(OCR1) $(OCR2) $(OCR3) $(OCR4) $(OCR5) $(OCR6))
+$(OCR8): TOOL = ocrd-calamari-recognize
+$(OCR8): GPU = 1
+$(OCR8): PARAMS = "checkpoint" : "$(VIRTUAL_ENV)/share/calamari/GT4HistOCR/*.ckpt.json"
+
+LINES = $(patsubst %,OCR-D-IMG-LINES-%,$(DEW) $(OCR1) $(OCR2) $(OCR3) $(OCR4) $(OCR5) $(OCR6) $(OCR7) $(OCR8))
 
 $(LINES): OCR-D-IMG-LINES-%: %
 $(LINES): TOOL = ocrd-segment-extract-lines
@@ -109,6 +116,9 @@ $(OUTPUT): $(LINES)
 
 .PHONY: $(OUTPUT)
 
+comma = ,
+concatcomma = $(subst $() $(),$(comma),$(1))
+
 # Down here, custom configuration ends.
 ###
 

gt-binarize-page-olena-sauvola-denoise-ocropy-deskew-page-ocropy-clip-deskew-region-tesseract-resegment-dewarp-ocr-ocropy-tesseract.mk

@@ -50,7 +50,7 @@ DESK2 = $(CLIP)-DESKEW-tesseract
 
 $(DESK2): $(CLIP)
 $(DESK2): TOOL = ocrd-tesserocr-deskew
-$(DESK2): PARAMS = "operation_level": "region"
+$(DESK2): PARAMS = "operation_level": "region", "min_orientation_confidence": 1.5
 
 RESEG = $(DESK2)-RESEG
 
@@ -85,6 +85,7 @@ $(OCR6): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model"
 $(OCR7): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "GT4HistOCR_2000000+GT4HistOCR_300000+GT4HistOCR_100000"
 
 $(OCR8): TOOL = ocrd-calamari-recognize
+$(OCR8): GPU = 1
 $(OCR8): PARAMS = "checkpoint" : "$(VIRTUAL_ENV)/share/calamari/GT4HistOCR/*.ckpt.json"
 
 OUTPUT = $(DEW)-OCR

gt-binarize-page-olena-sauvola-denoise-ocropy-deskew-page-ocropy-clip-resegment-dewarp-ocr-ocropy-tesseract.mk

@@ -79,6 +79,7 @@ $(OCR6): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model"
 $(OCR7): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "GT4HistOCR_2000000+GT4HistOCR_300000+GT4HistOCR_100000"
 
 $(OCR8): TOOL = ocrd-calamari-recognize
+$(OCR8): GPU = 1
 $(OCR8): PARAMS = "checkpoint" : "$(VIRTUAL_ENV)/share/calamari/GT4HistOCR/*.ckpt.json"
 
 OUTPUT = $(DEW)-OCR

gt-binarize-page-olena-sauvola-deskew-page-ocropy-clip-deskew-region-tesseract-resegment-dewarp-ocr-ocropy-tesseract.mk

@@ -44,7 +44,7 @@ DESK2 = $(CLIP)-DESKEW-tesseract
 
 $(DESK2): $(CLIP)
 $(DESK2): TOOL = ocrd-tesserocr-deskew
-$(DESK2): PARAMS = "operation_level": "region"
+$(DESK2): PARAMS = "operation_level": "region", "min_orientation_confidence": 1.5
 
 RESEG = $(DESK2)-RESEG
 
@@ -79,6 +79,7 @@ $(OCR6): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model"
 $(OCR7): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "GT4HistOCR_2000000+GT4HistOCR_300000+GT4HistOCR_100000"
 
 $(OCR8): TOOL = ocrd-calamari-recognize
+$(OCR8): GPU = 1
 $(OCR8): PARAMS = "checkpoint" : "$(VIRTUAL_ENV)/share/calamari/GT4HistOCR/*.ckpt.json"
 
 OUTPUT = $(DEW)-OCR

gt-binarize-page-olena-sauvola-deskew-page-ocropy-clip-resegment-dewarp-ocr-ocropy-tesseract.mk

@@ -73,6 +73,7 @@ $(OCR6): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model"
 $(OCR7): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "GT4HistOCR_2000000+GT4HistOCR_300000+GT4HistOCR_100000"
 
 $(OCR8): TOOL = ocrd-calamari-recognize
+$(OCR8): GPU = 1
 $(OCR8): PARAMS = "checkpoint" : "$(VIRTUAL_ENV)/share/calamari/GT4HistOCR/*.ckpt.json"
 
 OUTPUT = $(DEW)-OCR

gt-binarize-page-olena-wolf-denoise-ocropy-clip-resegment-dewarp-ocr-ocropy-tesseract.mk

@@ -73,6 +73,7 @@ $(OCR6): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model"
 $(OCR7): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "GT4HistOCR_2000000+GT4HistOCR_300000+GT4HistOCR_100000"
 
 $(OCR8): TOOL = ocrd-calamari-recognize
+$(OCR8): GPU = 1
 $(OCR8): PARAMS = "checkpoint" : "$(VIRTUAL_ENV)/share/calamari/GT4HistOCR/*.ckpt.json"
 
 OUTPUT = $(DEW)-OCR

gt-binarize-page-olena-wolf-denoise-ocropy-deskew-page-ocropy-clip-deskew-region-tesseract-resegment-dewarp-ocr-ocropy-tesseract.mk

@@ -50,7 +50,7 @@ DESK2 = $(CLIP)-DESKEW-tesseract
 
 $(DESK2): $(CLIP)
 $(DESK2): TOOL = ocrd-tesserocr-deskew
-$(DESK2): PARAMS = "operation_level": "region"
+$(DESK2): PARAMS = "operation_level": "region", "min_orientation_confidence": 1.5
 
 RESEG = $(DESK2)-RESEG
 
@@ -85,6 +85,7 @@ $(OCR6): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model"
 $(OCR7): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "GT4HistOCR_2000000+GT4HistOCR_300000+GT4HistOCR_100000"
 
 $(OCR8): TOOL = ocrd-calamari-recognize
+$(OCR8): GPU = 1
 $(OCR8): PARAMS = "checkpoint" : "$(VIRTUAL_ENV)/share/calamari/GT4HistOCR/*.ckpt.json"
 
 OUTPUT = $(DEW)-OCR

gt-binarize-page-olena-wolf-denoise-ocropy-deskew-page-ocropy-clip-resegment-dewarp-ocr-ocropy-tesseract.mk

@@ -79,6 +79,7 @@ $(OCR6): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model"
 $(OCR7): PARAMS = "textequiv_level" : "glyph", "overwrite_words": true, "model" : "GT4HistOCR_2000000+GT4HistOCR_300000+GT4HistOCR_100000"
 
 $(OCR8): TOOL = ocrd-calamari-recognize
+$(OCR8): GPU = 1
 $(OCR8): PARAMS = "checkpoint" : "$(VIRTUAL_ENV)/share/calamari/GT4HistOCR/*.ckpt.json"
 
 OUTPUT = $(DEW)-OCR