QA Spec

Makefile

@@ -1,10 +1,10 @@
-json: \
-	bagit-profile.json \
-	ocrd_tool.schema.json \
-	openapi.json
-
-deps:
-	pip install yaml click
+json: $(shell find -name '*.json')
 
 %.json: %.yml
 	python3 scripts/yaml-to-json.py $< $@
+
+validate: json
+	jsonschema --output pretty --validator Draft201909Validator --instance ocrd_eval.sample.json ocrd_eval.schema.json
+
+deps:
+	pip install yaml click jsonschema

bagit-profile.json

@@ -1 +1,49 @@
-{"BagIt-Profile-Info":{"BagIt-Profile-Identifier":"https://ocr-d.de/en/spec/bagit-profile.json","BagIt-Profile-Version":"1.2.0","Source-Organization":"OCR-D","External-Description":"BagIt profile for OCR data","Contact-Name":"Konstantin Baierer","Contact-Email":"konstantin.baierer@sbb.spk-berlin.de","Version":0.1},"Bag-Info":{"Bagging-Date":{"required":false},"Source-Organization":{"required":false},"Ocrd-Mets":{"required":false,"default":"mets.xml"},"Ocrd-Identifier":{"required":true},"Ocrd-Checksum":{"required":false,"default":"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"}},"Manifests-Required":["sha512"],"Tag-Manifests-Required":[],"Tag-Files-Required":[],"Tag-Files-Allowed":["README.md","Makefile","build.sh","sources.csv","metadata/*.xml","metadata/*.txt"],"Allow-Fetch.txt":false,"Serialization":"required","Accept-Serialization":"application/zip","Accept-BagIt-Version":["1.0"]}
+{
+  "BagIt-Profile-Info": {
+    "BagIt-Profile-Identifier": "https://ocr-d.de/en/spec/bagit-profile.json",
+    "BagIt-Profile-Version": "1.2.0",
+    "Source-Organization": "OCR-D",
+    "External-Description": "BagIt profile for OCR data",
+    "Contact-Name": "Konstantin Baierer",
+    "Contact-Email": "konstantin.baierer@sbb.spk-berlin.de",
+    "Version": 0.1
+  },
+  "Bag-Info": {
+    "Bagging-Date": {
+      "required": false
+    },
+    "Source-Organization": {
+      "required": false
+    },
+    "Ocrd-Mets": {
+      "required": false,
+      "default": "mets.xml"
+    },
+    "Ocrd-Identifier": {
+      "required": true
+    },
+    "Ocrd-Checksum": {
+      "required": false,
+      "default": "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"
+    }
+  },
+  "Manifests-Required": [
+    "sha512"
+  ],
+  "Tag-Manifests-Required": [],
+  "Tag-Files-Required": [],
+  "Tag-Files-Allowed": [
+    "README.md",
+    "Makefile",
+    "build.sh",
+    "sources.csv",
+    "metadata/*.xml",
+    "metadata/*.txt"
+  ],
+  "Allow-Fetch.txt": false,
+  "Serialization": "required",
+  "Accept-Serialization": "application/zip",
+  "Accept-BagIt-Version": [
+    "1.0"
+  ]
+}

ocrd_eval.sample.json

@@ -0,0 +1,120 @@
+[
+  {
+    "@id": "https://github.com/OCR-D/quiver/tree/data/evaluations/wf1-data345-eval1.json",
+    "label": "OCR workflow 1 on workspace 345",
+    "metadata": {
+      "ocr_workflow": {
+        "@id": "https://github.com/OCR-D/quiver/tree/data/workflows/1.nf",
+        "label": "OCR Workflow 1"
+      },
+      "eval_workflow": {
+        "@id": "https://github.com/OCR-D/quiver/tree/data/workflows/eval1.nf",
+        "label": "Evaluation Workflow 1"
+      },
+      "gt_workspace": {
+        "@id": "https://gt.ocr-d.de/workspace/789",
+        "label": "GT workspace 789 (19th century fraktur)"
+      },
+      "ocr_workspace": {
+        "@id": "https://github.com/OCR-D/quiver/tree/data/workspaces/3000.ocrd.zip",
+        "label": "OCR result workspace 3000"
+      },
+      "eval_workspace": {
+        "@id": "https://github.com/OCR-D/quiver/tree/data/workspaces/345.ocrd.zip",
+        "label": "Evaluation Workspace 345"
+      },
+      "workflow_steps": {
+        "0": "Processor A",
+        "1": "Processor B"
+      },
+      "workflow_model": "Fraktur_GT4HistOCR",
+      "document_metadata": {
+        "fonts": [
+          "antiqua",
+          "fraktur"
+        ],
+        "publication_century": "1800-1900",
+        "publication_decade": "1850-1860",
+        "publication_year": 1855,
+        "number_of_pages": 100,
+        "layout": "simple"
+      }
+    },
+    "evaluation": {
+      "document_wide": {
+        "wall_time": 1234,
+        "cer": 0.57,
+        "cer_min_max": [
+          0.2,
+          0.57
+        ]
+      },
+      "by_page": [
+        {
+          "page_id": "PHYS_0001",
+          "cer": 0.8,
+          "processing_time": 2.1
+        }
+      ]
+    }
+  },
+  {
+    "@id": "https://github.com/OCR-D/quiver/tree/data/evaluations/wf2-data345-eval1.json",
+    "label": "OCR Workflow 2 on Data 345",
+    "metadata": {
+      "ocr_workflow": {
+        "@id": "https://github.com/OCR-D/quiver/tree/data/workflows/2.nf",
+        "label": "OCR Workflow 2"
+      },
+      "eval_workflow": {
+        "@id": "https://github.com/OCR-D/quiver/tree/data/workflows/eval1.nf",
+        "label": "Evaluation Workflow 1"
+      },
+      "gt_workspace": {
+        "@id": "https://gt.ocr-d.de/workspace/789",
+        "label": "GT workspace 789 (19th century fraktur)"
+      },
+      "ocr_workspace": {
+        "@id": "https://github.com/OCR-D/quiver/tree/data/workspaces/3000.ocrd.zip",
+        "label": "OCR result workspace 3000"
+      },
+      "eval_workspace": {
+        "@id": "https://github.com/OCR-D/quiver/tree/data/workspaces/345.ocrd.zip",
+        "label": "Evaluation Workspace 345"
+      },
+      "workflow_steps": {
+        "0": "Processor A",
+        "1": "Processor B"
+      },
+      "workflow_model": "Fraktur_GT4HistOCR",
+      "document_metadata": {
+        "fonts": [
+          "antiqua",
+          "fraktur"
+        ],
+        "publication_century": "1800-1900",
+        "publication_decade": "1850-1860",
+        "publication_year": 1855,
+        "number_of_pages": 100,
+        "layout": "simple"
+      }
+    },
+    "evaluation": {
+      "document_wide": {
+        "wall_time": 4567,
+        "cer": 0.9,
+        "cer_min_max": [
+          0.2,
+          0.99
+        ]
+      },
+      "by_page": [
+        {
+          "page_id": "PHYS_0001",
+          "cer": 0.9,
+          "processing_time": 2.1
+        }
+      ]
+    }
+  }
+]

ocrd_eval.sample.yml

@@ -0,0 +1,85 @@
+- '@id': https://github.com/OCR-D/quiver/tree/data/evaluations/wf1-data345-eval1.json
+  label: OCR workflow 1 on workspace 345
+  metadata:
+    ocr_workflow:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workflows/1.nf
+      label: OCR Workflow 1
+    eval_workflow:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workflows/eval1.nf
+      label: Evaluation Workflow 1
+    gt_workspace:
+      '@id': https://gt.ocr-d.de/workspace/789
+      label: GT workspace 789 (19th century fraktur)
+    ocr_workspace:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workspaces/3000.ocrd.zip
+      label: OCR result workspace 3000
+    eval_workspace:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workspaces/345.ocrd.zip
+      label: Evaluation Workspace 345
+    workflow_steps:
+      '0': Processor A
+      '1': Processor B
+    workflow_model: Fraktur_GT4HistOCR
+    document_metadata:
+      fonts:
+      - antiqua
+      - fraktur
+      publication_century: 1800-1900
+      publication_decade: 1850-1860
+      publication_year: 1855
+      number_of_pages: 100
+      layout: simple
+  evaluation:
+    document_wide:
+      wall_time: 1234
+      cer: 0.57
+      cer_min_max:
+      - 0.2
+      - 0.57
+    by_page:
+    - page_id: PHYS_0001
+      cer: 0.8
+      processing_time: 2.1
+
+- '@id': https://github.com/OCR-D/quiver/tree/data/evaluations/wf2-data345-eval1.json
+  label: OCR Workflow 2 on Data 345
+  metadata:
+    ocr_workflow:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workflows/2.nf
+      label: OCR Workflow 2
+    eval_workflow:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workflows/eval1.nf
+      label: Evaluation Workflow 1
+    gt_workspace:
+      '@id': https://gt.ocr-d.de/workspace/789
+      label: GT workspace 789 (19th century fraktur)
+    ocr_workspace:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workspaces/3000.ocrd.zip
+      label: OCR result workspace 3000
+    eval_workspace:
+      '@id': https://github.com/OCR-D/quiver/tree/data/workspaces/345.ocrd.zip
+      label: Evaluation Workspace 345
+    workflow_steps:
+      '0': Processor A
+      '1': Processor B
+    workflow_model: Fraktur_GT4HistOCR
+    document_metadata:
+      fonts:
+      - antiqua
+      - fraktur
+      publication_century: 1800-1900
+      publication_decade: 1850-1860
+      publication_year: 1855
+      number_of_pages: 100
+      layout: simple
+  evaluation:
+    document_wide:
+      wall_time: 4567
+      cer: 0.9
+      cer_min_max:
+      - 0.2
+      - 0.99
+    by_page:
+    - page_id: PHYS_0001
+      cer: 0.9
+      processing_time: 2.1