Merge branch 'master' into IATI-Dashboard-468-traceability

git.sh

@@ -16,9 +16,17 @@ mkdir -p $GITOUT_DIR/gitaggregate
 mkdir -p $GITOUT_DIR/gitaggregate-dated
 
 
+
+cd helpers
+# Update codelist mapping, codelists and schemas
+echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Update codelist mapping"
+./get_codelist_mapping.sh
+echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Update codelists"
+./get_codelists.sh
+echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Update schemas"
+./get_schemas.sh
 # Build a JSON file of metadata for each CKAN publisher, and for each dataset published.
 # This is based on the data from the CKAN API
-cd helpers
 echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Running ckan.py"
 python ckan.py
 cd ..

helpers/get_codelists.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-for x in 105 202; do
+for x in 105 203; do
     i=$(echo $x | head -c 1)
     mkdir -p codelists/$i
     wget "http://iatistandard.org/$x/codelists/downloads/clv2/json/en/Version.json" -O codelists/$i/Version.json
@@ -10,4 +10,5 @@ for x in 105 202; do
     wget "http://iatistandard.org/$x/codelists/downloads/clv2/json/en/SectorCategory.json" -O codelists/$i/SectorCategory.json
     wget "http://iatistandard.org/$x/codelists/downloads/clv2/json/en/DocumentCategory.json" -O codelists/$i/DocumentCategory.json
     wget "http://iatistandard.org/$x/codelists/downloads/clv2/json/en/AidType.json" -O codelists/$i/AidType.json
+    wget "http://iatistandard.org/$x/codelists/downloads/clv2/json/en/BudgetNotProvided.json" -O codelists/$i/BudgetNotProvided.json
 done

helpers/transparency_indicator/reference_spend_data.csv

@@ -364,9 +364,10 @@ Calculate"
 "United Nations Central Emergency Response Fund (CERF)","cerf",461000000,"https://docs.unocha.org/sites/dms/CERF/CERF-Annual_Report_2014-Final-6.15.15-WEB.pdf",,"CERF allocations to global humanitarian funding for relief work, page 14 of report",,,,,,"USD",
 "United Nations Children's Fund (UNICEF)","unicef",4868000000,"http://www.un.org/en/ecosoc/qcpr/pdf/statistical_annex_tables_on_funding_flows_2014.xlsx",,"Table B-1, UNICEF",,,,,,"USD",
 "United Nations Development Programme","undp",4397347000,"http://www.un.org/en/ga/search/view_doc.asp?symbol=A/70/5/ADD.1",,"p.144 'Total Programme expenses'",,,,,,"USD",
-"United Nations Educational, Scientific and Cultural Organization (UNESCO)","unesco",626000000,"http://www.un.org/en/ecosoc/qcpr/pdf/statistical_annex_tables_on_funding_flows_2014.xlsx",,"Table B-1, UNESCO",,,,,,"USD",
+"United Nations Educational, Scientific and Cultural Organization (UNESCO)","unesco",626000000,"http://www.un.org/en/ecosoc/qcpr/pdf/statistical_annex_tables_on_funding_flows_2014.xlsx",,"Table B-1, UNESCO",590000000,"https://www.un.org/ecosoc/sites/www.un.org.ecosoc/files/files/en/qcpr/statistical_annex_tables_on_funding_flows_2015.xlsx",,,,"USD",
+"The United Nations High Commissioner for Refugees (UNHCR)","unhcr",3355000000,"http://reporting.unhcr.org/financial",,"UNHCR Global Focus, Financials (reference provided on 12/09/2018)",3295000000,"http://reporting.unhcr.org/financial",,"UNHCR Global Focus, Financials (reference provided on 12/09/2018)",,"USD",
 "United Nations Office for Project Services (UNOPS)","unops",1290000000,"http://www.un.org/en/ga/search/view_doc.asp?symbol=A/70/5/Add.11",,"Source provided directly - awaiting page numbers",1614000000,"https://www.unops.org/english/About/Executive-board/documents-for-sessions/Pages/Documents-for-2016.aspx",,"Document entitled 'DP/OPS/2016/2', p.1",,"USD",
-"United Nations Office for the Coordination of Humanitarian Affairs (OCHA)","unocha","NO DATA",,,,"NO DATA",,,,,,
+"United Nations Office for the Coordination of Humanitarian Affairs (OCHA)","unocha",274010586,"http://interactive.unocha.org/publication/2014_annualreport/mobile/index.html#p=54",,"p.50,OCHA Annual Report",283546639,"http://interactive.unocha.org/publication/2015_annualreport/#p=60",,"p.60 OCHA Annual report",,"USD",
 "United Nations Population Fund","unfpa",960000000,"http://www.un.org/en/ecosoc/qcpr/pdf/statistical_annex_tables_on_funding_flows_2014.xlsx",,"Table B-1, UNFPA",,,,,,"USD",
 "United Nations World Food Programme","wfp",4996000000,"http://www.un.org/en/ecosoc/qcpr/pdf/statistical_annex_tables_on_funding_flows_2014.xlsx",,"Table B-1, WFP",,,,,,"USD",
 "United States","unitedstates",11077640246.21,"DAC*",,"Sum of Department of Agriculture, Department of Treasury, Department of Defense, Peace Corps, State Department, African Development Foundation, Department of Health and Human Services, Millennium Challenge Corporation, Inter-American Development Foundation",8986772183.42,"DAC*",,"Sum of Department of Agriculture, Department of Treasury, Department of Defense, Peace Corps, State Department, African Development Foundation, Department of Health and Human Services, Millennium Challenge Corporation, Inter-American Development Foundation",,"USD",

stats/dashboard.py

@@ -99,7 +99,7 @@ def get_codelist_mapping(major_version):
 
 CODELISTS = {'1':{}, '2':{}}
 for major_version in ['1', '2']:
-    for codelist_name in ['Version', 'ActivityStatus', 'Currency', 'Sector', 'SectorCategory', 'DocumentCategory', 'AidType']:
+    for codelist_name in ['Version', 'ActivityStatus', 'Currency', 'Sector', 'SectorCategory', 'DocumentCategory', 'AidType', 'BudgetNotProvided']:
         CODELISTS[major_version][codelist_name] = set(c['code'] for c in json.load(open('helpers/codelists/{}/{}.json'.format(major_version, codelist_name)))['data'])
 
 
@@ -363,13 +363,7 @@ def element_versions(self):
     @returns_numberdict
     @memoize
     def _major_version(self):
-        # TODO: Refactor to use _version
-        parent = self.element.getparent()
-        if parent is None:
-            print('No parent of iati-activity, is this a test? Assuming version 1.xx')
-            return '1'
-        version = self.element.getparent().attrib.get('version')
-        if version and version.startswith('2.'):
+        if self._version().startswith('2.'):
             return '2'
         else:
             return '1'
@@ -382,14 +376,14 @@ def _version(self):
         if parent is None:
             print('No parent of iati-activity, is this a test? Assuming version 1.01')
             return '1.01'
-        version = self.element.getparent().attrib.get('version')
+        version = parent.attrib.get('version')
         if version and version in allowed_versions:
             return version
         else:
             return '1.01'
 
     @returns_numberdict
-    def ruleset_passes(self):
+    def _ruleset_passes(self):
         out = {}
         for ruleset_name in ['standard']:
             ruleset = json.load(open('helpers/rulesets/{0}.json'.format(ruleset_name)), object_pairs_hook=OrderedDict)
@@ -418,10 +412,16 @@ def activities(self):
     def hierarchies(self):
         return {self.element.attrib.get('hierarchy'):1}
 
+    def _budget_not_provided(self):
+        if self.element.attrib.get('budget-not-provided') is not None:
+            return int(self.element.attrib.get('budget-not-provided'))
+        else:
+            return None
+
     def by_hierarchy(self):
         out = {}
         for stat in ['activities', 'elements', 'elements_total',
-                     'forwardlooking_currency_year', 'forwardlooking_activities_current', 'forwardlooking_activities_with_budgets',
+                     'forwardlooking_currency_year', 'forwardlooking_activities_current', 'forwardlooking_activities_with_budgets', 'forwardlooking_activities_with_budget_not_provided',
                      'comprehensiveness', 'comprehensiveness_with_validation', 'comprehensiveness_denominators', 'comprehensiveness_denominator_default'
                      ]:
             out[stat] = copy.deepcopy(getattr(self, stat)())
@@ -893,6 +893,24 @@ def forwardlooking_activities_with_budgets(self, date_code_runs=None):
         return { year: int(self._forwardlooking_is_current(year) and year in budget_years and not bool(self._forwardlooking_exclude_in_calculations(year=year, date_code_runs=date_code_runs)))
                     for year in range(this_year, this_year+3) }
 
+    @returns_numberdict
+    def forwardlooking_activities_with_budget_not_provided(self, date_code_runs=None):
+        """
+        Number of activities with the budget_not_provided attribute for this year and the following 2 years.
+
+        Note activities excluded according if they meet the logic in _forwardlooking_exclude_in_calculations()
+
+        Input:
+          date_code_runs -- a date object for when this code is run
+        Returns:
+          dictionary containing years with binary value if this activity is current and has the budget_not_provided attribute
+        """
+        date_code_runs = date_code_runs if date_code_runs else self.now.date()
+        this_year = int(date_code_runs.year)
+        bnp = self._budget_not_provided() is not None
+        return {year: int(self._forwardlooking_is_current(year) and bnp > 0 and not bool(self._forwardlooking_exclude_in_calculations(year=year, date_code_runs=date_code_runs)))
+                for year in range(this_year, this_year+3)}
+
     @memoize
     def _comprehensiveness_is_current(self):
         """
@@ -998,10 +1016,10 @@ def is_text_in_element(elementName):
             return True if textFound else False
 
         return {
-            'version': (self.element.getparent() is not None
-                        and 'version' in self.element.getparent().attrib),
-            'reporting-org': (self.element.xpath('reporting-org/@ref')
-                        and is_text_in_element('reporting-org')),
+            'version': (self.element.getparent() is not None and
+                        'version' in self.element.getparent().attrib),
+            'reporting-org': (self.element.xpath('reporting-org/@ref') and
+                        is_text_in_element('reporting-org')),
             'iati-identifier': self.element.xpath('iati-identifier/text()'),
             'participating-org': self.element.find('participating-org') is not None,
             'title': is_text_in_element('title'),
@@ -1013,9 +1031,9 @@ def is_text_in_element(elementName):
                         for transaction in self.element.findall('transaction')
                 )),
             'country_or_region': (
-                self.element.find('recipient-country') is not None
-                or self.element.find('recipient-region') is not None
-                or (self._major_version() != '1' and all_true_and_not_empty(
+                self.element.find('recipient-country') is not None or
+                self.element.find('recipient-region') is not None or
+                (self._major_version() != '1' and all_true_and_not_empty(
                     (transaction.find('recipient-country') is not None or
                      transaction.find('recipient-region') is not None)
                         for transaction in self.element.findall('transaction')
@@ -1026,6 +1044,7 @@ def is_text_in_element(elementName):
             'transaction_traceability': all_true_and_not_empty(x.xpath('provider-org/@provider-activity-id') for x in self.element.xpath('transaction[transaction-type/@code="{}" or transaction-type/@code="11" or transaction-type/@code="13"]'.format(self._incoming_funds_code())))
                                         or self._is_donor_publisher(),
             'budget': self.element.findall('budget'),
+            'budget_not_provided': self._budget_not_provided() is not None,
             'contact-info': self.element.findall('contact-info/email'),
             'location': self.element.xpath('location/point/pos|location/name|location/description|location/location-administrative'),
             'location_point_pos': self.element.xpath('location/point/pos'),
@@ -1037,8 +1056,8 @@ def is_text_in_element(elementName):
             'conditions_attached': self.element.xpath('conditions/@attached'),
             'result_indicator': self.element.xpath('result/indicator'),
             'aid_type': (
-                all_true_and_not_empty(self.element.xpath('default-aid-type/@code'))
-                or all_true_and_not_empty([transaction.xpath('aid-type/@code') for transaction in self.element.xpath('transaction')])
+                all_true_and_not_empty(self.element.xpath('default-aid-type/@code')) or
+                all_true_and_not_empty([transaction.xpath('aid-type/@code') for transaction in self.element.xpath('transaction')])
                 )
             # Alternative: all(map(all_true_and_not_empty, [transaction.xpath('aid-type/@code') for transaction in self.element.xpath('transaction')]))
         }
@@ -1058,7 +1077,7 @@ def _is_sector_dac(self):
     def _comprehensiveness_with_validation_bools(self):
 
             def element_ref(element_obj):
-                """Get the ref attribute of a given element
+                """Get the ref attribute of a given element.
 
                 Returns:
                   Value in the 'ref' attribute or None if none found
@@ -1090,7 +1109,6 @@ def empty_or_percentage_sum_is_100(path, by_vocab=False):
                             for es in elements_by_vocab.values())
                     else:
                         return len(elements) == 1 or sum(decimal_or_zero(x.attrib.get('percentage')) for x in elements) == 100
-
             bools.update({
                 'version': bools['version'] and self.element.getparent().attrib['version'] in CODELISTS[self._major_version()]['Version'],
                 'iati-identifier': (
@@ -1136,6 +1154,9 @@ def empty_or_percentage_sum_is_100(path, by_vocab=False):
                         valid_date(budget.find('value')) and
                         valid_value(budget.find('value'))
                         for budget in bools['budget'])),
+                'budget_not_provided': (
+                    bools['budget_not_provided'] and
+                    str(self._budget_not_provided()) in CODELISTS[self._major_version()]['BudgetNotProvided']),
                 'location_point_pos': all_true_and_not_empty(
                     valid_coords(x.text) for x in bools['location_point_pos']),
                 'sector_dac': (
@@ -1205,7 +1226,7 @@ def comprehensiveness_denominators(self):
 
     @returns_numberdict
     def humanitarian(self):
-        humanitarian_sectors_dac_5_digit = ['72010', '72040', '72050', '73010', '74010']
+        humanitarian_sectors_dac_5_digit = ['72010', '72040', '72050', '73010', '74010', '74020']
         humanitarian_sectors_dac_3_digit = ['720', '730', '740']
 
         # logic around use of the @humanitarian attribute
@@ -1234,8 +1255,17 @@ def humanitarian(self):
         return {
             'is_humanitarian': is_humanitarian,
             'is_humanitarian_by_attrib': is_humanitarian_by_attrib,
-            'contains_humanitarian_scope': 1 if (self._version() in ['2.02', '2.03']) and self.element.xpath('humanitarian-scope/@type') and self.element.xpath('humanitarian-scope/@code') else 0,
-            'uses_humanitarian_clusters_vocab': 1 if (self._version() in ['2.02', '2.03']) and self.element.xpath('sector/@vocabulary="10"') else 0
+            'contains_humanitarian_scope': 1 if (
+                is_humanitarian and
+                self._version() in ['2.02', '2.03'] and
+                all_true_and_not_empty(self.element.xpath('humanitarian-scope/@type')) and
+                all_true_and_not_empty(self.element.xpath('humanitarian-scope/@code'))
+            ) else 0,
+            'uses_humanitarian_clusters_vocab': 1 if (
+                is_humanitarian and
+                self._version() in ['2.02', '2.03'] and
+                self.element.xpath('sector/@vocabulary="10"')
+            ) else 0
         }
 
     def _transaction_type_code(self, transaction):

stats/tests/test_budget_not_provided.py

@@ -0,0 +1,37 @@
+from lxml import etree
+
+from stats.dashboard import ActivityStats
+
+class MockActivityStats(ActivityStats):
+    def __init__(self, major_version='2'):
+        self.major_version = major_version
+        return super(MockActivityStats, self).__init__()
+
+    def _major_version(self):
+        return self.major_version
+
+def test_budget_not_provided_works():
+    activity_stats = MockActivityStats()
+    activity_stats.element = etree.fromstring('''
+            <iati-activity budget-not-provided="1">
+            </iati-activity>
+    ''')
+    assert activity_stats._budget_not_provided() == 1
+
+
+def test_budget_not_provided_fails():
+    activity_stats = MockActivityStats()
+    activity_stats.element = etree.fromstring('''
+            <iati-activity>
+            </iati-activity>
+    ''')
+    assert activity_stats._budget_not_provided() is None
+
+
+def test_budget_validation_bools():
+    activity_stats = MockActivityStats()
+    activity_stats.element = etree.fromstring('''
+            <iati-activity budget-not-provided="3">
+            </iati-activity>
+    ''')
+    assert (len(activity_stats.element.findall('budget')) == 0)

stats/tests/test_comprehensiveness.py

@@ -185,6 +185,7 @@ def test_comprehensiveness_empty(major_version):
         'transaction_currency': 0,
         'transaction_traceability': 0,
         'budget': 0,
+        'budget_not_provided': 0,
         'contact-info': 0,
         'location': 0,
         'location_point_pos': 0,
@@ -319,6 +320,7 @@ def test_comprehensiveness_full(major_version):
         'transaction_currency': 1,
         'transaction_traceability': 1,
         'budget': 1,
+        'budget_not_provided': 0,
         'contact-info': 1,
         'location': 1,
         'location_point_pos': 1,
@@ -393,6 +395,7 @@ def test_comprehensiveness_other_passes(major_version):
         'transaction_currency': 1,
         'transaction_traceability': 0,
         'budget': 0,
+        'budget_not_provided': 0,
         'contact-info': 0,
         'location': 0,
         'location_point_pos': 0,
@@ -1514,6 +1517,7 @@ def test_comprehensiveness_dac_sector_codes_v2(major_version):
         'transaction_currency': 1,
         'transaction_traceability': 1,
         'budget': 1,
+        'budget_not_provided': 0,
         'contact-info': 1,
         'location': 1,
         'location_point_pos': 1,
@@ -1605,6 +1609,7 @@ def test_comprehensiveness_dac_sector_codes_v2_incomplete(major_version):
         'transaction_currency': 1,
         'transaction_traceability': 1,
         'budget': 1,
+        'budget_not_provided': 0,
         'contact-info': 1,
         'location': 1,
         'location_point_pos': 1,
@@ -1689,6 +1694,7 @@ def test_comprehensiveness_v1_returns_false(major_version):
         'transaction_currency': 1,
         'transaction_traceability': 1,
         'budget': 1,
+        'budget_not_provided': 0,
         'contact-info': 1,
         'location': 1,
         'location_point_pos': 1,