Add script to check for common model issues

.github/workflows/build.yml

@@ -29,6 +29,9 @@ jobs:
           sudo apt-get install -y -qq libicu-dev
           pip install wheel pyicu
           pip install -e ".[dev]"
+      - name: Run checks for default model
+        run: |
+          python contrib/check_model.py
       - name: Run the tests
         run: |
           make test

contrib/check_model.py

@@ -0,0 +1,131 @@
+import sys
+from collections import defaultdict
+from followthemoney import model
+
+
+IGNORE_DIVERGENT_TYPES = [
+    "author",
+    "organization",
+    "gender",
+    "number",
+    "authority",
+    "duration",
+    "area",
+    "subject",
+    "sender",
+]
+
+IGNORE_DIVERGENT_LABELS = [
+    "parent",
+    "holder",
+    "number",
+    "authority",
+    "criteria",
+    "procedure",
+]
+
+IGNORE_LABEL_COLLISIONS = [
+    "Address",
+    "Notes",
+    "Customs declarations",
+    "Country of origin",
+    "Payments received",
+    "Payments made",
+    "Entity",
+    "The language of the translated text",
+    "Responding to",
+    "ISIN",
+    "Document number",
+]
+
+
+def test_divergent_types(by_name):
+    divergent = {}
+
+    for name, props in by_name.items():
+        if len(props) == 1 or name in IGNORE_DIVERGENT_TYPES:
+            continue
+
+        types = set([p.type for p in props])
+        if len(types) > 1:
+            divergent[name] = props
+
+    return divergent
+
+
+def test_divergent_labels(by_name):
+    divergent = {}
+
+    for name, props in by_name.items():
+        if len(props) == 1 or name in IGNORE_DIVERGENT_LABELS:
+            continue
+
+        labels = set([p.label for p in props])
+        if len(labels) > 1:
+            divergent[name] = props
+
+    return divergent
+
+
+def test_label_collisions(by_label):
+    collisions = {}
+
+    for label, props in by_label.items():
+        if len(props) == 1 or label in IGNORE_LABEL_COLLISIONS:
+            continue
+
+        names = set([p.name for p in props])
+        if len(names) > 1:
+            collisions[label] = props
+
+    return collisions
+
+
+if __name__ == '__main__':
+    by_name = defaultdict(set)
+    by_label = defaultdict(set)
+
+    for schema in model:
+        for prop in schema.properties.values():
+            by_name[prop.name].add(prop)
+            by_label[prop.label].add(prop)
+
+    divergent_types = test_divergent_types(by_name)
+    divergent_labels = test_divergent_labels(by_name)
+    label_collisions = test_label_collisions(by_label)
+
+    failed = False
+
+    if divergent_types:
+        failed = True
+        print("DIVERGENT TYPES\n")
+        for name, props in divergent_types.items():
+            print(f"  {name}:")
+            for prop in props:
+                print(f"  * {prop.qname} - {prop.type.name}")
+            print()
+        print()
+
+    if divergent_labels:
+        failed = True
+        print("DIVERGENT LABELS\n")
+        for name, props in divergent_labels.items():
+            print(f"  {name}:")
+            for prop in props:
+                print(f"  * {prop.qname} - {prop.label}")
+            print()
+        print()
+
+    if label_collisions:
+        failed = True
+        print("COLLIDING LABELS\n")
+        for label, props in label_collisions.items():
+            print(f"  {label}:")
+            for prop in props:
+                print(f"  * {prop.qname}")
+            print()
+
+    if failed:
+        sys.exit(1)
+
+    print("No issues found.")

followthemoney/schema/CallForTenders.yaml

@@ -54,14 +54,14 @@ CallForTenders:
       label: "Call for tenders result"
       description: "The nature of the contractual agreement that will result from this CfT"
     cpvCode:
-      label: "CPV codes"
+      label: "CPV code"
       description: "Common Procurement Vocabulary (CPV)"
       type: identifier
     reverseAuctionsIncluded:
       label: "Inclusion of e-Auctions"
       # cf. https://mita.gov.mt/2021/10/05/reverse-bidding-for-data-connectivity-across-the-public-service-a-first-for-malta/
     nutsCode:
-      label: "NUTS codes"
+      label: "NUTS code"
       description: "Nomenclature of Territorial Units for Statistics (NUTS)"
       type: identifier
     relationToThreshold:

followthemoney/schema/Contract.yaml

@@ -22,7 +22,7 @@ Contract:
       - contractDate
   properties:
     title:
-      label: "Contract title"
+      label: "Title"
       type: string
     authority:
       label: "Contract authority"

followthemoney/schema/ContractAward.yaml

@@ -40,7 +40,7 @@ ContractAward:
       type: entity
       range: Contract
     callForTenders:
-      label: "Call For Tender"
+      label: "Call For Tenders"
       type: entity
       reverse:
         name: contractAwards
@@ -58,9 +58,11 @@ ContractAward:
     cpvCode:
       label: "CPV code"
       description: "Contract Procurement Vocabulary (what type of goods/services, EU)"
+      type: identifier
     nutsCode:
       label: "NUTS code"
       description: "Nomencalture of Territorial Units for Statistics (NUTS)"
+      type: identifier
     amended:
       label: "Amended"
       description: "Was this award amended, modified or updated by a subsequent document?"

followthemoney/schema/Post.yaml

@@ -35,6 +35,6 @@ Post:
       label: "Organization"
       type: string
     wikidataId:
-      label: "Wikidata ID of the post"
+      label: "Wikidata ID"
       hidden: true
       type: identifier

followthemoney/schema/Security.yaml

@@ -27,7 +27,7 @@ Security:
       label: Registration number
       type: identifier
     ticker:
-      label: Ticker
+      label: Stock ticker symbol
       type: identifier
     issuer:
       label: "Issuer"