[#834] iati_orgids: add orgids check option to cli

README.rst

@@ -265,8 +265,11 @@ Options:
 
 ``--delete -d`` Delete the output directory if it already exists.
 
+``--orgids -i`` Check IATI identifier prefixes against Org-ids prefixes.
+
 ``--openag -a`` Run ruleset checks for IATI OpenAg data.
 
+
 If the file is in spreadsheet format, the output directory will contain a *unflattened.xml* file converted from Excel or CSV to XML format
 
 

cove_iati/lib/api.py

@@ -30,7 +30,7 @@ def context_api_transform(context):
     return context
 
 
-def iati_json_output(output_dir, file, openag=False):
+def iati_json_output(output_dir, file, openag=False, orgids=False):
     context = {}
     file_type = get_file_type(file)
     context = {"file_type": file_type}
@@ -47,8 +47,8 @@ def iati_json_output(output_dir, file, openag=False):
         data_file = file
 
     context = context_api_transform(
-        common_checks_context_iati(context, output_dir, data_file,
-                                   file_type, api=True, openag=openag)
+        common_checks_context_iati(context, output_dir, data_file, file_type,
+                                   api=True, openag=openag, orgids=orgids)
     )
 
     if file_type != 'xml':
@@ -61,7 +61,8 @@ def iati_json_output(output_dir, file, openag=False):
             shutil.rmtree(os.path.join(output_dir, 'csv_dir'))
 
     ruleset_dirs = [os.path.join(output_dir, 'ruleset'),
-                    os.path.join(output_dir, 'ruleset_openag')]
+                    os.path.join(output_dir, 'ruleset_openag'),
+                    os.path.join(output_dir, 'ruleset_orgids')]
     for directory in ruleset_dirs:
         if os.path.exists(directory):
             shutil.rmtree(directory)

cove_iati/lib/iati.py

@@ -11,7 +11,7 @@
 from cove.lib.exceptions import CoveInputDataError, UnrecognisedFileTypeXML
 
 
-def common_checks_context_iati(context, upload_dir, data_file, file_type, api=False, openag=False):
+def common_checks_context_iati(context, upload_dir, data_file, file_type, api=False, openag=False, orgids=False):
     schema_aiti = SchemaIATI()
     lxml_errors = {}
     cell_source_map = {}
@@ -50,6 +50,10 @@ def common_checks_context_iati(context, upload_dir, data_file, file_type, api=Fa
             ruleset_errors_ag = get_openag_ruleset_errors(tree, os.path.join(upload_dir, 'ruleset_openag'))
             context.update({'ruleset_errors_openag': ruleset_errors_ag})
 
+        if orgids:
+            ruleset_errors_orgids = get_orgids_ruleset_errors(tree, os.path.join(upload_dir, 'ruleset_orgids'))
+            context.update({'ruleset_errors_orgids': ruleset_errors_orgids})
+
     errors_all = format_lxml_errors(lxml_errors)
 
     if file_type != 'xml':
@@ -277,6 +281,15 @@ def get_openag_ruleset_errors(lxml_etree, output_dir):
     return format_ruleset_errors(output_dir)
 
 
+def get_orgids_ruleset_errors(lxml_etree, output_dir):
+    bdd_tester(etree=lxml_etree, features=['cove_iati/rulesets/iati_orgids_ruleset/'],
+               output_path=output_dir)
+
+    if not os.path.isdir(output_dir):
+        return []
+    return format_ruleset_errors(output_dir)
+
+
 def get_file_type(file):
     if isinstance(file, str):
         name = file.lower()

cove_iati/management/commands/iati_cli.py

@@ -13,14 +13,17 @@ class Command(CoveBaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--openag', '-a', action='store_true', help='Run ruleset checks for IATI OpenAg')
+        parser.add_argument('--orgids', '-i', action='store_true', help='Check IATI identifier prefixes against '
+                            'Org-ids prefixes')
         super(Command, self).add_arguments(parser)
 
     def handle(self, file, *args, **options):
         super(Command, self).handle(file, *args, **options)
         openag = options.get('openag')
+        orgids = options.get('orgids')
 
         try:
-            result = iati_json_output(self.output_dir, file, openag=openag)
+            result = iati_json_output(self.output_dir, file, openag=openag, orgids=orgids)
         except APIException as e:
             self.stdout.write(str(e))
             sys.exit(1)

cove_iati/rulesets/iati_openag_ruleset/steps/openag_ruleset_step_definitions.py

@@ -3,15 +3,10 @@
 Released under MIT License
 License: https://github.com/pwyf/bdd-tester/blob/master/LICENSE
 '''
-import re
-
 from behave import then
 
-from cove.lib.common import get_orgids_prefixes
 from cove_iati.rulesets.utils import get_full_xpath, get_xpaths, register_ruleset_errors
 
-ORGIDS_PREFIXES = get_orgids_prefixes()
-
 
 @then('at least one `{xpath_expression}` element is expected')
 @register_ruleset_errors('openag')
@@ -67,20 +62,3 @@ def step_openag_location_id_expected(context, xpath_expression1, xpath_expressio
             errors.append({'message': fail_msg.format(xpath_expression1, xpath_expression2),
                            'path': get_full_xpath(context.xml, xpath)})
     return context, errors
-
-
-@then('`{attribute}` id attribute must start with an org-ids prefix')
-@register_ruleset_errors('openag')
-def step_openag_org_id_prefix_expected(context, attribute):
-    errors = []
-    fail_msg = '@{} {} does not start with a recognised org-ids prefix'
-
-    for xpath in get_xpaths(context.xml, context.xpath_expression):
-        attr_id = xpath.attrib.get(attribute, '')
-        for prefix in ORGIDS_PREFIXES:
-            if re.match('^%s' % prefix, attr_id):
-                break
-        else:
-            errors.append({'message': fail_msg.format(attribute, attr_id),
-                           'path': '{}/@{}'.format(get_full_xpath(context.xml, xpath), attribute)})
-    return context, errors

cove_iati/rulesets/iati_orgids_ruleset/steps/orgids_step_definitions.py

@@ -0,0 +1,25 @@
+import re
+
+from behave import then
+
+from cove.lib.common import get_orgids_prefixes
+from cove_iati.rulesets.utils import get_full_xpath, get_xpaths, register_ruleset_errors
+
+ORGIDS_PREFIXES = get_orgids_prefixes()
+
+
+@then('`{attribute}` id attribute must start with an org-ids prefix')
+@register_ruleset_errors()
+def step_openag_org_id_prefix_expected(context, attribute):
+    errors = []
+    fail_msg = '@{} {} does not start with a recognised org-ids prefix'
+
+    for xpath in get_xpaths(context.xml, context.xpath_expression):
+        attr_id = xpath.attrib.get(attribute, '')
+        for prefix in ORGIDS_PREFIXES:
+            if re.match('^%s' % prefix, attr_id):
+                break
+        else:
+            errors.append({'message': fail_msg.format(attribute, attr_id),
+                           'path': '{}/@{}'.format(get_full_xpath(context.xml, xpath), attribute)})
+    return context, errors

cove_iati/tests.py

@@ -304,6 +304,7 @@ def test_cove_iati_cli_output():
         results = json.load(fp)
 
     assert not results.get('ruleset_errors_openag')
+    assert not results.get('ruleset_errors_orgids')
 
     validation_errors = results.get('validation_errors')
     validation_errors.sort(key=lambda i: i['path'])
@@ -328,6 +329,48 @@ def test_cove_iati_cli_output():
             assert expected['message'] == actual['message']
 
 
+def test_cove_iati_cli_orgids_output():
+    expected = [{'id': '?TZ-BRLA-5-CCC-123123-CC123',
+                 'message': '@ref  does not start with a recognised org-ids prefix',
+                 'path': '/iati-activities/iati-activity[3]/participating-org/@ref',
+                 'rule': 'participating-org/@ref must have an org-ids prefix'},
+                {'id': '?TZ-BRLA-5-CCC-123123-CC123',
+                 'message': '@ref ?TZ-BRLA-5 does not start with a recognised org-ids prefix',
+                 'path': '/iati-activities/iati-activity[3]/reporting-org/@ref',
+                 'rule': 'reporting-org/@ref must have an org-ids prefix'},
+                {'id': 'TZ-BRLA-5-DDD-123123-DD123',
+                 'message': '@ref ?TZ-BRLA-8 does not start with a recognised org-ids prefix',
+                 'path': '/iati-activities/iati-activity[4]/participating-org/@ref',
+                 'rule': 'participating-org/@ref must have an org-ids prefix'},
+                {'id': 'TZ-BRLA-9-EEE-123123-EE123',
+                 'message': '@ref ?TZ-BRLA-101 does not start with a recognised org-ids prefix',
+                 'path': '/iati-activities/iati-activity[5]/transaction[1]/provider-org/@ref',
+                 'rule': 'transaction/provider-org/@ref must have an org-ids prefix'},
+                {'id': 'TZ-BRLA-9-EEE-123123-EE123',
+                 'message': '@ref ?TZ-BRLA-102 does not start with a recognised org-ids prefix',
+                 'path': '/iati-activities/iati-activity[5]/transaction[2]/receiver-org/@ref',
+                 'rule': 'transaction/receiver-org/@ref must have an org-ids prefix'}]
+
+    file_path = os.path.join('cove_iati', 'fixtures', 'basic_iati_ruleset_errors.xml')
+    output_dir = os.path.join('media', str(uuid.uuid4()))
+    call_command('iati_cli', file_path, output_dir=output_dir, orgids=True)
+
+    with open(os.path.join(output_dir, 'results.json')) as fp:
+        results = json.load(fp)
+
+    assert not results.get('ruleset_errors_openag')
+
+    ruleset_errors = results.get('ruleset_errors_orgids')
+    ruleset_errors.sort(key=lambda i: i['path'])
+    zipped_results = zip(expected, ruleset_errors)
+
+    for expected, actual in zipped_results:
+        assert expected['id'] == actual['id']
+        assert expected['message'] == actual['message']
+        assert expected['path'] == actual['path']
+        assert expected['rule'] == actual['rule']
+
+
 def test_cove_iati_cli_openag_output():
     expected = [{'id': 'AA-AAA-123123-AA123',
                  'message': 'the activity should include at least one location element',
@@ -338,10 +381,6 @@ def test_cove_iati_cli_openag_output():
                  'path': '/iati-activities/iati-activity[1]/openag:tag',
                  'rule': 'openag:tag/@vocabulary must be present with a code for "maintained by the '
                          'reporting organisation"'},
-                {'id': 'AA-AAA-123123-AA123',
-                 'message': '@ref NO-ORGIDS-10000 does not start with a recognised org-ids prefix',
-                 'path': '/iati-activities/iati-activity[1]/reporting-org/@ref',
-                 'rule': 'reporting-org/@ref must have an org-ids prefix'},
                 {'id': 'BB-BBB-123123-BB123',
                  'message': 'location/location-id element must have @code attribute',
                  'path': '/iati-activities/iati-activity[2]/location/location-id',
@@ -351,10 +390,6 @@ def test_cove_iati_cli_openag_output():
                             'should be "http://aims.fao.org/aos/agrovoc/")',
                  'path': '/iati-activities/iati-activity[2]/openag:tag/@vocabulary-uri',
                  'rule': 'openag:tag/@vocabulary-uri must be present with an agrovoc uri'},
-                {'id': 'BB-BBB-123123-BB123',
-                 'message': '@ref NO-ORGIDS-40000 does not start with a recognised org-ids prefix',
-                 'path': '/iati-activities/iati-activity[2]/participating-org/@ref',
-                 'rule': 'participating-org/@ref must have an org-ids prefix'},
                 {'id': 'CC-CCC-789789-CC789',
                  'message': 'location/location-id element must have @vocabulary attribute',
                  'path': '/iati-activities/iati-activity[3]/location/location-id',
@@ -384,6 +419,8 @@ def test_cove_iati_cli_openag_output():
     with open(os.path.join(output_dir, 'results.json')) as fp:
         results = json.load(fp)
 
+    assert not results.get('ruleset_errors_orgids')
+
     ruleset_errors = results.get('ruleset_errors_openag')
     ruleset_errors.sort(key=lambda i: i['path'])
     zipped_results = zip(expected, ruleset_errors)