Merge pull request #856 from OpenDataServices/834-additional-iati-checks

834 additional iati checks

README.rst

@@ -265,6 +265,13 @@ Options:
 
 ``--delete -d`` Delete the output directory if it already exists.
 
+``--orgids -i`` Run org-ids rule check for IATI identifier prefixes.
+
+``--openag -a`` Run ruleset checks for IATI OpenAg data.
+
+
 If the file is in spreadsheet format, the output directory will contain a *unflattened.xml* file converted from Excel or CSV to XML format
 
+**OpenaAg** rulesets check that the data contains the XML elements ``<opeang:tag>`` and ``<location>``, and that they include the right attributes expected for OpenAg data. Please read `OpenAg ruleset feature files <cove_iati/rulesets/iati_openag_ruleset/>`_ (written in `Gerkhin <https://github.com/cucumber/cucumber/wiki/Gherkin/>`_ style) for more information.
 
+**Org-ids** rulesets check that all organisation identifiers are prefixed with a registered `org-ids <http://org-id.guide>`_ prefix. Please read `Org-ids ruleset feature file <cove_iati/rulesets/iati_orgids_ruleset/>`_ for more information

cove/lib/common.py

@@ -1,5 +1,6 @@
 import collections
 import csv
+import datetime
 import functools
 import json
 import os
@@ -631,3 +632,37 @@ def get_spreadsheet_meta_data(upload_dir, file_name, schema, file_type='xlsx', n
     with open(output_name) as metatab_data:
         metatab_json = json.load(metatab_data)
     return metatab_json
+
+
+def get_orgids_prefixes(orgids_url=None):
+    local_org_ids_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'org-ids.json')
+    today = datetime.date.today()
+    get_remote_file = False
+    first_request = False
+
+    if not orgids_url:
+        orgids_url = 'http://org-id.guide/download.json'
+
+    if os.path.exists(local_org_ids_file):
+        with open(local_org_ids_file) as fp:
+            org_ids = json.load(fp)
+        date_str = org_ids.get('downloaded', '2000-1-1')
+        date_downloaded = datetime.datetime.strptime(date_str, '%Y-%m-%d').date()
+        if date_downloaded != today:
+            get_remote_file = True
+    else:
+        get_remote_file = True
+        first_request = True
+
+    if get_remote_file:
+        try:
+            org_ids = requests.get(orgids_url).json()
+            org_ids['downloaded'] = "%s" % today
+            with open(local_org_ids_file, 'w') as fp:
+                json.dump(org_ids, fp, indent=2)
+        except requests.exceptions.RequestException:
+            if first_request:
+                raise  # First time ever request fails
+            pass  # Update fails
+
+    return [org_list['code'] for org_list in org_ids['lists']]

cove_iati/fixtures/SOURCES

@@ -0,0 +1,5 @@
+example.xml
+Source: http://iatistandard.org/202/activity-standard/example-xml/
+Copyright © Development Initiatives, on behalf of the IATI Secretariat
+Released under the Creative Commons attribution license
+http://iatistandard.org/202/license/

cove_iati/fixtures/basic_iati_ruleset_errors.xml

@@ -1,63 +1,195 @@
-<iati-activities version="2.01">
+<iati-activities version="2.02">
   <iati-activity>
-    <iati-identifier>AA-AAA-123123-AA123</iati-identifier>
-    <reporting-org ref="AA-AAA-123" type="40"><narrative>Reporting Organisation A</narrative></reporting-org>
+    <iati-identifier>TZ-BRLA-1-AAA-123123-AA123</iati-identifier>
+    <reporting-org ref="TZ-BRLA-1" type="40">
+      <narrative>Reporting Organisation 1</narrative>
+    </reporting-org>
     <title><narrative>A title</narrative></title>
     <description><narrative>A description</narrative></description>
-    <participating-org role="1" ref="AA-AAA-123"/>
+    <participating-org role="1" ref="TZ-BRLA-2"/>
     <activity-status code="3"/>
-    <recipient-country code="AF" percentage="30"/>
-    <recipient-country code="XK" percentage="60"/>
-    <sector code='1'/>
+    <activity-date type="2" iso-date="22000101"/>
+    <recipient-country code="TZ" percentage="100"/>
+    <sector vocabulary="1" code="1"/>
+    <budget>
+      <period-start iso-date="2016-01-01"/>
+      <period-end iso-date="2017-01-01"/>
+    </budget>
     <transaction>
       <transaction-type code="2"/>
+      <transaction-date iso-date="22000101"/>
+      <value value-date="2017-01-01">20</value>
     </transaction>
     <transaction>
       <transaction-type code="3"/>
-      <transaction-date iso-date="2200-03-03"/>
-      <value value-date="2012-03-03">20</value>
+      <transaction-date iso-date="2200-01-01"/>
+      <value value-date="2017-01-01">20</value>
     </transaction>
     <transaction>
       <transaction-type code="2"/>
-      <transaction-date iso-date="2013-04-04"/>
-      <value value-date="2013-04-04">30</value>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">20</value>
     </transaction>
     <transaction>
       <transaction-type code="3"/>
-      <transaction-date iso-date="2013-05-05"/>
-      <value value-date="2014-05-05" >40</value>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01" >40</value>
     </transaction>
   </iati-activity>
   <iati-activity>
-    <iati-identifier>BB-BBB-456456-BB456</iati-identifier>
-    <reporting-org ref="BB-BBB-56789" type="40"><narrative>Reporting Organisation B</narrative></reporting-org>
+    <iati-identifier>TZ-BRLA-3-BBB-123123-BB123</iati-identifier>
+    <reporting-org ref="TZ-BRLA-3" type="40">
+      <narrative>Reporting Organisation 3</narrative>
+    </reporting-org>
     <title><narrative>A title</narrative></title>
     <description><narrative>A description</narrative></description>
-    <participating-org role="1" ref="BB-BBB-456"/>
+    <participating-org role="1" ref="TZ-BRLA-4"/>
+    <activity-status code="3"/>
+    <activity-date type="2" iso-date="2200-01-01"/>
+    <recipient-country code="TZ" percentage="100"/>
+    <budget>
+      <period-start iso-date="20150101"/>
+      <period-end iso-date="20140101"/>
+    </budget>
+    <transaction>
+      <transaction-type code="2"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="24000101">10</value>
+      <sector vocabulary="1" code="1"/>
+    </transaction>
+    <transaction>
+      <transaction-type code="2"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2200-01-01">20</value>
+      <sector vocabulary="1" code="1" />
+    </transaction>
+    <transaction>
+      <transaction-type code="2"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">30</value>
+      <sector vocabulary="1" code="1"/>
+    </transaction>
+    <transaction>
+      <transaction-type code="3"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">40</value>
+      <sector vocabulary="1" code="1"/>
+    </transaction>
+  </iati-activity>
+  <iati-activity>
+    <iati-identifier>?TZ-BRLA-5-CCC-123123-CC123</iati-identifier>
+    <reporting-org ref="?TZ-BRLA-5" type="40">
+      <narrative>Reporting Organisation 5</narrative>
+    </reporting-org>
+    <title><narrative>A title</narrative></title>
+    <description>
+      <narrative>A description</narrative>
+    </description>
+    <participating-org />
     <activity-status code="3"/>
-    <activity-date type="2" iso-date="2013-01-01"/>
+    <activity-date type="2" iso-date="22000101"/>
+    <activity-date type="4" iso-date="24000101"/>
     <recipient-country code="AF" percentage="30"/>
-    <recipient-country code="XK" percentage="60"/>
-    <sector code='1'/>
+    <sector vocabulary="1" code="1"/>
+    <budget>
+      <period-start iso-date="2010-01-01"/>
+      <period-end iso-date="2009-01-01"/>
+    </budget>
     <transaction>
       <transaction-type code="2"/>
-      <transaction-date iso-date="2012-01-01"/>
-      <value value-date="2013-01-01">10</value>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">10</value>
+      <sector vocabulary="1" code="1"/>
+    </transaction>
+    <transaction>
+      <transaction-type code="3"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">20</value>
+      <sector vocabulary="1" code="1"/>
     </transaction>
     <transaction>
       <transaction-type code="2"/>
-      <transaction-date iso-date="2012-03-03"/>
-      <value value-date="2012-03-03">20</value>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">30</value>
+    </transaction>
+    <transaction>
+      <transaction-type code="3"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">40</value>
+    </transaction>
+  </iati-activity>
+  <iati-activity>
+    <iati-identifier>TZ-BRLA-5-DDD-123123-DD123</iati-identifier>
+    <reporting-org ref="TZ-BRLA-7" type="40">
+      <narrative>Reporting Organisation 5</narrative>
+    </reporting-org>
+    <title><narrative>A title</narrative></title>
+    <description>
+      <narrative>A description</narrative>
+    </description>
+    <participating-org role="1" ref="?TZ-BRLA-8"/>
+    <activity-status code="3"/>
+    <activity-date type="4" iso-date="2400-01-01"/>
+    <recipient-country code="AF" percentage="30"/>
+    <sector vocabulary="1" code="1"/>
+    <transaction>
+      <transaction-type code="2"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2012-01-01">10</value>
+      <provider-org ref="TZ-BRLA-801"/>
+    </transaction>
+    <transaction>
+      <transaction-type code="3"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">20</value>
+      <receiver-org ref="TZ-BRLA-802"/>
+    </transaction>
+    <transaction>
+      <transaction-type code="2"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">30</value>
+    </transaction>
+    <transaction>
+      <transaction-type code="3"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">40</value>
+    </transaction>
+  </iati-activity>
+  <iati-activity>
+    <iati-identifier>TZ-BRLA-9-EEE-123123-EE123</iati-identifier>
+    <reporting-org ref="TZ-BRLA-9" type="40">
+      <narrative>Reporting Organisation 9</narrative>
+    </reporting-org>
+    <title><narrative>A title</narrative></title>
+    <description>
+      <narrative>A description</narrative>
+    </description>
+    <participating-org role="1" ref="TZ-BRLA-10"/>
+    <activity-status code="3"/>
+    <activity-date iso-date="2017-01-01"/>
+    <recipient-country code="AF" percentage="30"/>
+    <sector vocabulary="1" code="1"/>
+    <transaction>
+      <transaction-type code="2"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2012-01-01">10</value>
+      <provider-org ref="?TZ-BRLA-101"/>
+    </transaction>
+    <transaction>
+      <transaction-type code="3"/>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">20</value>
+      <receiver-org ref="?TZ-BRLA-102"/>
     </transaction>
     <transaction>
       <transaction-type code="2"/>
-      <transaction-date iso-date="2013-04-04"/>
-      <value value-date="2013-04-04">30</value>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">30</value>
     </transaction>
     <transaction>
       <transaction-type code="3"/>
-      <transaction-date iso-date="2013-05-05"/>
-      <value value-date="2013-05-05">40</value>
+      <transaction-date iso-date="2017-01-01"/>
+      <value value-date="2017-01-01">40</value>
     </transaction>
   </iati-activity>
 </iati-activities>