[#2042] IATI import bugs

akvo/iati/imports/__init__.py

@@ -79,9 +79,9 @@ def __init__(self, iati_import_job, parent_elem, project, globals,
     def get_or_create_organisation(self, ref='', name='', org_id=None, internal_id=None):
         """
         Looks for an organisation in the RSR database.
-        First the ref will be looked up in the Organisation.iati_org_id field. If this does not exist,
-        the name will be looked up in the Organisation.name and Organisation.long_name fields.
-        If none of these return a match, a new organisation will be created.
+        First the ref will be looked up in the Organisation.iati_org_id field. If this does not
+        exist, the name will be looked up in the Organisation.name and Organisation.long_name
+        fields. If none of these return a match, a new organisation will be created.
 
         :param ref: String; the reference of the organisation that is specified in the IATI file.
         :param name: String; the name of the organisation that is specified in the IATI file.
@@ -117,7 +117,8 @@ def get_or_create_organisation(self, ref='', name='', org_id=None, internal_id=N
             name=name[:25],
             long_name=name[:75],
             iati_org_id=ref if ref else None,
-            organisation_type=ORG_TYPE_NGO
+            organisation_type=ORG_TYPE_NGO,
+            content_owner=self.project.reporting_org
         )
 
     def do_import(self):
@@ -143,14 +144,14 @@ def add_log(self, tag, field, error, message_type=LOG_ENTRY_TYPE.CRITICAL_ERROR)
         """
 
         self._log_objects += [IatiImportLog(
-                iati_import_job=self.iati_import_job,
-                tag=tag,
-                model=self.model,
-                field=field,
-                text=error,
-                project=self.project,
-                message_type=message_type,
-                created_at=datetime.now(),
+            iati_import_job=self.iati_import_job,
+            tag=tag,
+            model=self.model,
+            field=field,
+            text=error,
+            project=self.project,
+            message_type=message_type,
+            created_at=datetime.now(),
         )]
 
     def get_text(self, element):

akvo/iati/imports/mappers/CordaidZip/organisations.py

@@ -14,6 +14,8 @@
 from ... import ImportMapper
 from . import same_data
 
+class NotOwnedOrganisationException(Exception):
+    pass
 
 class InternalOrganisationIDs(ImportMapper):
 
@@ -64,17 +66,17 @@ def set_logo(self, organisation, identifier):
                 return 'logo'
         return None
 
-    def update_organisation(self, organisation, fields_data):
+    def update_organisation(self, organisation, fields):
         """
         Update the organisation
         :param organisation: the organisation to update
-        :param fields_data: dict with new fields data
+        :param fields: dict with new fields data
         :return: list; the fields that were updated
         """
         changes = []
-        for field in fields_data:
-            if getattr(organisation, field) != fields_data[field]:
-                setattr(organisation, field, fields_data[field])
+        for field in fields:
+            if getattr(organisation, field) != fields[field]:
+                setattr(organisation, field, fields[field])
                 changes.append(field)
         if changes:
             organisation.save(update_fields=changes)
@@ -87,6 +89,7 @@ def do_import(self):
         that way we need to create it before we can create the InternalOrganisationID object.
         """
 
+        CORDAID_ORG_ID = 273
         ioids = InternalOrganisationIDs(
                 self.iati_import_job, self.parent_elem, self.project, self.globals)
         identifier = ioids.get_child_element_text(ioids.parent_elem, 'org_id', 'identifier')
@@ -99,19 +102,32 @@ def do_import(self):
                 referenced_org = None
         else:
             referenced_org = None
+        if referenced_org:
+            owner = referenced_org.content_owner
+            if owner and owner != self.globals['cordaid']:
+                raise NotOwnedOrganisationException(
+                        "Organisation {}, ID {}, is content owned by {}, ID {}. "
+                        "Can't edit the data.".format(
+                                referenced_org.name, referenced_org.pk,
+                                owner.name, owner.pk
+                        ))
+
         org_fields = {}
         org_fields['long_name'] = self.get_child_element_text(
                 self.parent_elem, 'name', 'long_name').strip()
         org_fields['name'] = org_fields['long_name'][:25]
         org_fields['new_organisation_type'] = int(self.get_child_element_text(
                 self.parent_elem, 'iati_organisation_type', 'new_organisation_type', 22))
         org_fields['iati_org_id'] = self.get_child_element_text(
-                self.parent_elem, 'iati_org_id', 'iati_org_id')
+            self.parent_elem, 'iati_org_id', 'iati_org_id')
+        org_fields['description'] = self.get_child_element_text(
+            self.parent_elem, 'description', 'description')
         # TODO: validate URL
         org_fields['url'] = self.get_child_element_text(self.parent_elem, 'url', 'url')
+        org_fields['content_owner'] = self.globals['cordaid']
 
         if referenced_org:
-            organisation = Organisation.objects.get(pk=referenced_org.pk)
+            organisation = referenced_org
             changes = self.update_organisation(organisation, org_fields)
             created = False
         else:

akvo/iati/imports/mappers/defaults.py

@@ -242,5 +242,6 @@ def do_import(self):
         """
 
         humanitarian = self.get_attrib(self.parent_elem, 'humanitarian', 'humanitarian', None)
-        humanitarian = self.to_boolean(humanitarian)
+        if humanitarian:
+            humanitarian = self.to_boolean(humanitarian)
         return self.update_project_field('humanitarian', humanitarian)

akvo/iati/imports/mappers/locations.py

@@ -5,6 +5,7 @@
 # For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
 
 from ....rsr.models.country import Country, RecipientCountry
+from ....rsr.models.iati_import_log import LOG_ENTRY_TYPE
 from ....rsr.models.location import AdministrativeLocation, ProjectLocation
 from ....rsr.models.region import RecipientRegion
 
@@ -87,7 +88,8 @@ def do_import(self):
                 try:
                     country = Country.objects.get(iso_code=country_code)
                 except ObjectDoesNotExist as e:
-                    self.add_log('administrative', 'country', str(e))
+                    self.add_log('administrative', 'country', str(e),
+                                 LOG_ENTRY_TYPE.VALUE_NOT_SAVED)
 
             loc, created = ProjectLocation.objects.get_or_create(
                 location_target=self.project,

akvo/rsr/migrations/0069_auto_20160517_1515.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import akvo.rsr.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rsr', '0068_iaticheck'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='project',
+            name='background',
+            field=akvo.rsr.fields.ValidXMLTextField(help_text='This should describe the geographical, political, environmental, social and/or cultural context of the project, and any related activities that have already taken place or are underway. For links and styling of the text, <a href="https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet" target="_blank">Markdown</a> is supported.', verbose_name='background', blank=True),
+            preserve_default=True,
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='current_status',
+            field=akvo.rsr.fields.ValidXMLTextField(help_text='Describe the situation at the start of the project. For links and styling of the text, <a href="https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet" target="_blank">Markdown</a> is supported.', verbose_name='baseline situation', blank=True),
+            preserve_default=True,
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='goals_overview',
+            field=akvo.rsr.fields.ValidXMLTextField(help_text='Provide a brief description of the overall project goals. For links and styling of the text, <a href="https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet" target="_blank">Markdown</a> is supported.', verbose_name='goals overview', blank=True),
+            preserve_default=True,
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='project_plan',
+            field=akvo.rsr.fields.ValidXMLTextField(help_text='Detailed information about the implementation of the project: the what, how, who and when. For links and styling of the text, <a href="https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet" target="_blank">Markdown</a> is supported.', verbose_name='project plan', blank=True),
+            preserve_default=True,
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='sustainability',
+            field=akvo.rsr.fields.ValidXMLTextField(help_text='Describe how you aim to guarantee sustainability of the project until 10 years after project implementation. Think about the institutional setting, capacity-building, a cost recovery plan, products used, feasible arrangements for operation and maintenance, anticipation of environmental impact and social integration. For links and styling of the text, <a href="https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet" target="_blank">Markdown</a> is supported.', verbose_name='sustainability', blank=True),
+            preserve_default=True,
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='target_group',
+            field=akvo.rsr.fields.ProjectLimitedTextField(help_text='This should include information about the people, organisations or resources that are being impacted by this project. For links and styling of the text, <a href="https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet" target="_blank">Markdown</a> is supported.', verbose_name='target group', blank=True),
+            preserve_default=True,
+        ),
+        migrations.AlterField(
+            model_name='projectdocument',
+            name='title',
+            field=akvo.rsr.fields.ValidXMLCharField(default='Untitled document', help_text='Enter the title of your document.', max_length=100, verbose_name='document title', blank=True),
+            preserve_default=True,
+        ),
+        migrations.AlterField(
+            model_name='projectlocation',
+            name='activity_description',
+            field=akvo.rsr.fields.ValidXMLCharField(help_text='A description that qualifies the activity taking place at the location. This should not duplicate information provided in the main activity description, and should typically be used to distinguish between activities at multiple locations within a single iati-activity record.', max_length=2000, verbose_name='activity description', blank=True),
+            preserve_default=True,
+        ),
+        migrations.AlterField(
+            model_name='projectlocation',
+            name='description',
+            field=akvo.rsr.fields.ValidXMLCharField(help_text='This provides free text space for providing an additional description, if needed, of the actual target of the activity. A description that qualifies the location, not the activity.', max_length=2000, verbose_name='location description', blank=True),
+            preserve_default=True,
+        ),
+        migrations.AlterField(
+            model_name='sector',
+            name='sector_code',
+            field=akvo.rsr.fields.ValidXMLCharField(help_text='Please select DAC-5 or DAC-3 as the sector vocabulary first, then this field will be populated with the corresponding codes. For other vocabularies, it is possible to fill in any code. See these lists for the DAC-5 and DAC-3 sector codes: <a href="http://iatistandard.org/202/codelists/Sector/" target="_blank">DAC-5 sector codes</a> and <a href="http://iatistandard.org/202/codelists/SectorCategory/" target="_blank">DAC-3 sector codes</a>.', max_length=25, verbose_name='sector code', blank=True),
+            preserve_default=True,
+        ),
+    ]

akvo/rsr/models/country.py

@@ -69,7 +69,10 @@ class RecipientCountry(models.Model):
 
     def __unicode__(self):
         if self.country:
-            country_unicode = self.iati_country().name
+            try:
+                country_unicode = self.iati_country().name
+            except (AttributeError, codelist_models.Country.DoesNotExist):
+                country_unicode = self.country
         else:
             country_unicode = u'%s' % _(u'No country specified')
 

akvo/rsr/models/iati_import_job.py

@@ -421,7 +421,7 @@ class CordaidZipIatiImportJob(IatiImportJob):
     the use of the ZIP archive to extract all information needed, both for the importing of
     organisations and activities.
     Note that self.iati_xml_file is used to hold the whole Cordaid ZIP archive, not just the
-    activities XML. The XML is instead put in self._iati_xml_file in check_file()
+    activities XML. The XML is instead injected directly into self.parse_xml()
     """
     IATI_XML_ACTIVITIES = 'iati-activities'
 
@@ -437,6 +437,7 @@ def import_organisations(self):
         ORGANISATIONS_ROOT = 'Relations'
         ORGANISATIONS_CHILDREN = 'object'
         CORDAID_ORG_ID = 273
+
         self.add_log(u'CordaidZip: Starting organisations import.', LOG_ENTRY_TYPE.INFORMATIONAL)
         organisations_xml = self.get_xml_file(ORGANISATIONS_FILENAME)
         if self.parse_xml(organisations_xml, ORGANISATIONS_ROOT, ORGANISATIONS_CHILDREN):
@@ -445,21 +446,29 @@ def import_organisations(self):
             organisations = None
         cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
         if organisations:
-            for object in organisations.findall(ORGANISATIONS_CHILDREN):
+            created_count = 0
+            updated_count = 0
+            for element in organisations.findall(ORGANISATIONS_CHILDREN):
                 try:
-                    org_mapper = Organisations(self, object, None, {'cordaid': cordaid})
+                    org_mapper = Organisations(self, element, None, {'cordaid': cordaid})
                     organisation, changes, created = org_mapper.do_import()
                     if created:
                         self.log_creation(organisation)
+                        created_count += 1
                     else:
                         self.log_changes(organisation, changes)
+                        if changes:
+                            updated_count += 1
                 except Exception as e:
                     self.add_log(
                             u'CordaidZip: Critical error when importing '
                             u'organisations: {}'.format(e.message),
                             LOG_ENTRY_TYPE.CRITICAL_ERROR)
 
-        self.add_log(u'CordaidZip: Organisations import done.', LOG_ENTRY_TYPE.INFORMATIONAL)
+        self.add_log(u'CordaidZip: Organisations import done. '
+                     u'{} organisations created, {} organisations updated'.format(
+                            created_count, updated_count),
+                     LOG_ENTRY_TYPE.INFORMATIONAL)
 
     def create_log_entry(self, organisation, action_flag=LOG_ENTRY_TYPE.ACTION_UPDATE,
                          change_message=u''):
@@ -507,13 +516,17 @@ def parse_xml(self, xml_file, root_tag='', children_tag=''):
         :return: True if all's well, False if not
         """
         try:
-            parsed_xml = etree.parse(xml_file)
+            parser = etree.XMLParser()
+            for line in xml_file:
+                parser.feed(line)
+                if line.strip() == "</{}>".format(root_tag):
+                    break
+            objects_root = parser.close()
         except Exception as e:
             self.add_log('Error parsing XML file. Error message:\n{}'.format(e.message),
                          LOG_ENTRY_TYPE.CRITICAL_ERROR)
             return False
 
-        objects_root = parsed_xml.getroot()
         if objects_root.tag == root_tag:
             self._objects_root = objects_root
             self.add_log(
@@ -552,7 +565,8 @@ def set_activities(self):
         :return: True or False indicating success or failure
         """
         IATI_XML_ACTIVITY = 'iati-activity'
-        if self.parse_xml(self._iati_xml_file, self.IATI_XML_ACTIVITIES, IATI_XML_ACTIVITY):
+
+        if self.parse_xml(self.get_activities_file(), self.IATI_XML_ACTIVITIES, IATI_XML_ACTIVITY):
             self.activities = self._objects_root
             return True
         else:
@@ -569,7 +583,10 @@ def check_file(self):
         if file:
             # Since we're using self.iati_xml_file for the Cordaid ZIP archive, we assign the IATI
             # XML to self._iati_xml_file
-            self._iati_xml_file = file
+            # self._iati_xml_file = file
+            # The above line doesn't work for some reason. For reasons not understood, when assiging
+            # the file to self._iat_xml_file the file isn't readable any more when we access is in
+            # parse_xml(). Instead we inject the file directly in the call to parse_xml()
             self.add_log('Using iati-activities.xml from the Cordaid ZIP: {}'.format(self.iati_xml_file),
                          LOG_ENTRY_TYPE.INFORMATIONAL)
             return True

akvo/rsr/models/location.py

@@ -99,13 +99,13 @@ class ProjectLocation(BaseLocation):
         help_text=_(u'The human-readable name for the location.')
     )
     description = ValidXMLCharField(
-        _(u'location description'), blank=True, max_length=255,
+        _(u'location description'), blank=True, max_length=2000,
         help_text=_(u'This provides free text space for providing an additional description, if '
                     u'needed, of the actual target of the activity. A description that qualifies '
                     u'the location, not the activity.')
     )
     activity_description = ValidXMLCharField(
-        _(u'activity description'), blank=True, max_length=255,
+        _(u'activity description'), blank=True, max_length=2000,
         help_text=_(u'A description that qualifies the activity taking place at the location. '
                     u'This should not duplicate information provided in the main activity '
                     u'description, and should typically be used to distinguish between activities '