checking references duplicates

ckanext/datajson/datajsonvalidator.py

@@ -89,13 +89,14 @@
 import os
 
 omb_burueau_codes = set()
-#for row in csv.DictReader(urllib.urlopen("https://project-open-data.cio.gov/data/omb_bureau_codes.csv")):
+# for row in csv.DictReader(urllib.urlopen("https://project-open-data.cio.gov/data/omb_bureau_codes.csv")):
 #    omb_burueau_codes.add(row["Agency Code"] + ":" + row["Bureau Code"])
 
 with open(os.path.join(os.path.dirname(__file__), "resources", "omb_bureau_codes.csv"), "r") as csvfile:
     for row in csv.DictReader(csvfile):
         omb_burueau_codes.add(row["Agency Code"] + ":" + row["Bureau Code"])
 
+
 # main function for validation
 def do_validation(doc, errors_array, seen_identifiers):
     errs = {}
@@ -376,6 +377,10 @@ def do_validation(doc, errors_array, seen_identifiers):
                         add_error(errs, 50, "Invalid Field Value (Optional Fields)",
                                   "The field 'references' had an invalid rfc3987 URL: \"%s\"" % s, dataset_name)
 
+                if len(item["references"]) != len(set(item["references"])):
+                    add_error(errs, 50, "Invalid Field Value (Optional Fields)",
+                              "The field 'references' has duplicates", dataset_name)
+
             # systemOfRecords # optional
             check_url_field(False, item, "systemOfRecords", dataset_name, errs, allow_redacted=True)