For records, check that ocids are unique, not whole records

Additionally, for the case where ids/ocids are missing, we fallback to the message "Array has non-unique elements" This ensures we don't get a block of python repr data in the error message: OpenDataServices/cove#1220 There's also been a change of case from "Non-unique ID Values" to "Non-unique id values".
OpenDataServices · Oct 24, 2019 · 558fe20 · 558fe20
1 parent 74bf3cb
commit 558fe20
Show file tree

Hide file tree

Showing 9 changed files with 576 additions and 136 deletions.
diff --git a/libcove/lib/common.py b/libcove/lib/common.py
@@ -56,12 +56,19 @@
 
 
 def unique_ids(validator, ui, instance, schema):
+    # `records` key from the JSON schema doesn't get passed through to here, so
+    # we look out for this $ref — this may change if the way the schema files
+    # are structured changes.
+    if schema.get("items") == {"$ref": "#/definitions/record"}:
+        id_name = "ocid"
+    else:
+        id_name = "id"
     if ui and validator.is_type(instance, "array"):
         non_unique_ids = set()
         all_ids = set()
         for item in instance:
             try:
-                item_id = item.get("id")
+                item_id = item.get(id_name)
             except AttributeError:
                 # if item is not a dict
                 item_id = None
@@ -74,14 +81,12 @@ def unique_ids(validator, ui, instance, schema):
                     non_unique_ids.add(item_id)
                 all_ids.add(item_id)
             else:
-                # if there is any item without an id key, or the item is not a dict
-                # revert to original validator
-                for error in uniqueItemsValidator(validator, ui, instance, schema):
-                    yield error
+                msg = "Array has non-unique elements"
+                yield ValidationError(msg, instance=instance)
                 return
 
-        for non_unique_id in non_unique_ids:
-            msg = "Non-unique ID Values"
+        for non_unique_id in sorted(non_unique_ids):
+            msg = "Non-unique {} values".format(id_name)
             yield ValidationError(msg, instance=non_unique_id)
 
 

diff --git a/tests/lib/fixtures/common/records_no_validation_errors.json b/tests/lib/fixtures/common/records_no_validation_errors.json
@@ -0,0 +1,17 @@
+{
+    "version": "1.1",
+    "publishedDate": "2019-09-18T17:56:21.078Z",
+    "publisher": {
+        "name": "A Publisher Name"
+    },
+    "uri": "http://example.org",
+    "records": [
+        {
+            "ocid": "EXAMPLE-1",
+            "releases": [{
+                "url": "http://example.org/",
+                "date": "2019-09-18T17:56:21.078Z"
+            }]
+        }
+    ]
+}
diff --git a/tests/lib/fixtures/common/records_non_unique.json b/tests/lib/fixtures/common/records_non_unique.json
@@ -0,0 +1,45 @@
+{
+    "version": "1.1",
+    "publishedDate": "2019-09-18T17:56:21.078Z",
+    "publisher": {
+        "name": "A Publisher Name"
+    },
+    "uri": "http://example.org",
+    "records": [
+        {
+            "ocid": "EXAMPLE-1",
+            "releases": [{
+                "url": "http://example.org/",
+                "date": "2019-09-18T17:56:21.078Z"
+            }]
+        },
+        {
+            "ocid": "EXAMPLE-1",
+            "releases": [{
+                "url": "http://example.org/",
+                "date": "2019-09-18T17:56:21.078Z"
+            }]
+        },
+        {
+            "ocid": "EXAMPLE-2",
+            "releases": [{
+                "url": "http://example.org/",
+                "date": "2019-09-18T17:56:21.078Z"
+            }]
+        },
+        {
+            "ocid": "EXAMPLE-2",
+            "releases": [{
+                "url": "http://example.org/",
+                "date": "2019-09-18T17:56:21.078Z"
+            }]
+        },
+        {
+            "ocid": "EXAMPLE-2",
+            "releases": [{
+                "url": "http://example.org/different_url",
+                "date": "2019-09-18T17:56:21.078Z"
+            }]
+        }
+    ]
+}
diff --git a/tests/lib/fixtures/common/records_non_unique_no_ocid.json b/tests/lib/fixtures/common/records_non_unique_no_ocid.json
@@ -0,0 +1,22 @@
+{
+    "version": "1.1",
+    "publishedDate": "2019-09-18T17:56:21.078Z",
+    "publisher": {
+        "name": "A Publisher Name"
+    },
+    "uri": "http://example.org",
+    "records": [
+        {
+            "releases": [{
+                "url": "http://example.org/",
+                "date": "2019-09-18T17:56:21.078Z"
+            }]
+        },
+        {
+            "releases": [{
+                "url": "http://example.org/",
+                "date": "2019-09-18T17:56:21.078Z"
+            }]
+        }
+    ]
+}
diff --git a/tests/lib/fixtures/common/release-package-schema.json b/tests/lib/fixtures/common/release-package-schema.json
@@ -0,0 +1,111 @@
+{
+  "id": "https://standard.open-contracting.org/schema/1__1__4/release-package-schema.json",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Schema for an Open Contracting Release Package",
+  "description": "The release package contains a list of releases along with some publishing metadata.",
+  "type": "object",
+  "required": [
+    "uri",
+    "publisher",
+    "publishedDate",
+    "releases",
+    "version"
+  ],
+  "properties": {
+    "uri": {
+      "title": "Package identifier",
+      "description": "The URI of this package that identifies it uniquely in the world. Recommended practice is to use a dereferenceable URI, where a persistent copy of this package is available.",
+      "type": "string",
+      "format": "uri"
+    },
+    "version": {
+      "title": "OCDS schema version",
+      "description": "The version of the OCDS schema used in this package, expressed as major.minor For example: 1.0 or 1.1",
+      "type": "string",
+      "pattern": "^(\\d+\\.)(\\d+)$"
+    },
+    "extensions": {
+      "title": "OCDS extensions",
+      "description": "An array of OCDS extensions used in this package, in which each array item is the URL of an extension.json file.",
+      "type": "array",
+      "items": {
+        "type": "string",
+        "format": "uri"
+      }
+    },
+    "publishedDate": {
+      "title": "Published date",
+      "description": "The date that this package was published. If this package is generated 'on demand', this date should reflect the date of the last change to the underlying contents of the package.",
+      "type": "string",
+      "format": "date-time"
+    },
+    "releases": {
+      "title": "Releases",
+      "description": "An array of one or more OCDS releases.",
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "$ref": "https://standard.open-contracting.org/schema/1__1__4/release-schema.json"
+      },
+      "uniqueItems": true
+    },
+    "publisher": {
+      "title": "Publisher",
+      "description": "Information to uniquely identify the publisher of this package.",
+      "type": "object",
+      "properties": {
+        "name": {
+          "title": "Name",
+          "description": "The name of the organization or department responsible for publishing this data.",
+          "type": "string"
+        },
+        "scheme": {
+          "title": "Scheme",
+          "description": "The scheme that holds the unique identifiers used to identify the item being identified.",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "uid": {
+          "title": "uid",
+          "description": "The unique ID for this entity under the given ID scheme.",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "uri": {
+          "title": "URI",
+          "description": "A URI to identify the publisher.",
+          "type": [
+            "string",
+            "null"
+          ],
+          "format": "uri"
+        }
+      },
+      "required": [
+        "name"
+      ]
+    },
+    "license": {
+      "title": "License",
+      "description": "A link to the license that applies to the data in this package. A Public Domain Dedication or [Open Definition Conformant](http://opendefinition.org/licenses/) license is recommended. The canonical URI of the license should be used. Documents linked from this file may be under other license conditions.",
+      "type": [
+        "string",
+        "null"
+      ],
+      "format": "uri"
+    },
+    "publicationPolicy": {
+      "title": "Publication policy",
+      "description": "A link to a document describing the publishers [publication policy](https://standard.open-contracting.org/1.1/en/implementation/publication_policy/).",
+      "type": [
+        "string",
+        "null"
+      ],
+      "format": "uri"
+    }
+  }
+}
diff --git a/tests/lib/fixtures/common/releases_no_validation_errors.json b/tests/lib/fixtures/common/releases_no_validation_errors.json
@@ -0,0 +1,17 @@
+{
+    "version": "1.1",
+    "publishedDate": "2019-09-18T17:56:21.078Z",
+    "publisher": {
+        "name": "A Publisher Name"
+    },
+    "uri": "http://example.org",
+    "releases": [
+        {
+            "id": "EXAMPLE-1-1",
+            "ocid": "EXAMPLE-1",
+            "date": "2019-09-18T17:56:21.078Z",
+            "initiationType": "tender",
+            "tag": ["planning"]
+        }
+    ]
+}
diff --git a/tests/lib/fixtures/common/releases_non_unique.json b/tests/lib/fixtures/common/releases_non_unique.json
@@ -0,0 +1,45 @@
+{
+    "version": "1.1",
+    "publishedDate": "2019-09-18T17:56:21.078Z",
+    "publisher": {
+        "name": "A Publisher Name"
+    },
+    "uri": "http://example.org",
+    "releases": [
+        {
+            "id": "EXAMPLE-1-1",
+            "ocid": "EXAMPLE-1",
+            "date": "2019-09-18T17:56:21.078Z",
+            "initiationType": "tender",
+            "tag": ["planning"]
+        },
+        {
+            "id": "EXAMPLE-1-1",
+            "ocid": "EXAMPLE-1",
+            "date": "2019-09-18T17:56:21.078Z",
+            "initiationType": "tender",
+            "tag": ["planning"]
+        },
+        {
+            "id": "EXAMPLE-1-2",
+            "ocid": "EXAMPLE-1",
+            "date": "2019-09-18T17:56:21.078Z",
+            "initiationType": "tender",
+            "tag": ["planning"]
+        },
+        {
+            "id": "EXAMPLE-1-2",
+            "ocid": "EXAMPLE-1",
+            "date": "2019-09-18T17:56:21.078Z",
+            "initiationType": "tender",
+            "tag": ["planning"]
+        },
+        {
+            "id": "EXAMPLE-1-2",
+            "ocid": "EXAMPLE-1",
+            "date": "1999-09-18T17:56:21.078Z",
+            "initiationType": "tender",
+            "tag": ["planning"]
+        }
+    ]
+}
diff --git a/tests/lib/fixtures/common/releases_non_unique_no_id.json b/tests/lib/fixtures/common/releases_non_unique_no_id.json
@@ -0,0 +1,22 @@
+{
+    "version": "1.1",
+    "publishedDate": "2019-09-18T17:56:21.078Z",
+    "publisher": {
+        "name": "A Publisher Name"
+    },
+    "uri": "http://example.org",
+    "releases": [
+        {
+            "ocid": "EXAMPLE-1",
+            "date": "2019-09-18T17:56:21.078Z",
+            "initiationType": "tender",
+            "tag": ["planning"]
+        },
+        {
+            "ocid": "EXAMPLE-1",
+            "date": "2019-09-18T17:56:21.078Z",
+            "initiationType": "tender",
+            "tag": ["planning"]
+        }
+    ]
+}