diff --git a/akvo/iati/imports/__init__.py b/akvo/iati/imports/__init__.py
index 776a5795dd..2c72d04bcb 100644
--- a/akvo/iati/imports/__init__.py
+++ b/akvo/iati/imports/__init__.py
@@ -79,9 +79,9 @@ def __init__(self, iati_import_job, parent_elem, project, globals,
def get_or_create_organisation(self, ref='', name='', org_id=None, internal_id=None):
"""
Looks for an organisation in the RSR database.
- First the ref will be looked up in the Organisation.iati_org_id field. If this does not exist,
- the name will be looked up in the Organisation.name and Organisation.long_name fields.
- If none of these return a match, a new organisation will be created.
+ First the ref will be looked up in the Organisation.iati_org_id field. If this does not
+ exist, the name will be looked up in the Organisation.name and Organisation.long_name
+ fields. If none of these return a match, a new organisation will be created.
:param ref: String; the reference of the organisation that is specified in the IATI file.
:param name: String; the name of the organisation that is specified in the IATI file.
@@ -117,7 +117,8 @@ def get_or_create_organisation(self, ref='', name='', org_id=None, internal_id=N
name=name[:25],
long_name=name[:75],
iati_org_id=ref if ref else None,
- organisation_type=ORG_TYPE_NGO
+ organisation_type=ORG_TYPE_NGO,
+ content_owner=self.project.reporting_org
)
def do_import(self):
@@ -143,14 +144,14 @@ def add_log(self, tag, field, error, message_type=LOG_ENTRY_TYPE.CRITICAL_ERROR)
"""
self._log_objects += [IatiImportLog(
- iati_import_job=self.iati_import_job,
- tag=tag,
- model=self.model,
- field=field,
- text=error,
- project=self.project,
- message_type=message_type,
- created_at=datetime.now(),
+ iati_import_job=self.iati_import_job,
+ tag=tag,
+ model=self.model,
+ field=field,
+ text=error,
+ project=self.project,
+ message_type=message_type,
+ created_at=datetime.now(),
)]
def get_text(self, element):
diff --git a/akvo/iati/imports/mappers/CordaidZip/organisations.py b/akvo/iati/imports/mappers/CordaidZip/organisations.py
index 9a98033742..69a540503c 100644
--- a/akvo/iati/imports/mappers/CordaidZip/organisations.py
+++ b/akvo/iati/imports/mappers/CordaidZip/organisations.py
@@ -14,6 +14,8 @@
from ... import ImportMapper
from . import same_data
+class NotOwnedOrganisationException(Exception):
+ pass
class InternalOrganisationIDs(ImportMapper):
@@ -64,17 +66,17 @@ def set_logo(self, organisation, identifier):
return 'logo'
return None
- def update_organisation(self, organisation, fields_data):
+ def update_organisation(self, organisation, fields):
"""
Update the organisation
:param organisation: the organisation to update
- :param fields_data: dict with new fields data
+ :param fields: dict with new fields data
:return: list; the fields that were updated
"""
changes = []
- for field in fields_data:
- if getattr(organisation, field) != fields_data[field]:
- setattr(organisation, field, fields_data[field])
+ for field in fields:
+ if getattr(organisation, field) != fields[field]:
+ setattr(organisation, field, fields[field])
changes.append(field)
if changes:
organisation.save(update_fields=changes)
@@ -87,6 +89,7 @@ def do_import(self):
that way we need to create it before we can create the InternalOrganisationID object.
"""
+ CORDAID_ORG_ID = 273
ioids = InternalOrganisationIDs(
self.iati_import_job, self.parent_elem, self.project, self.globals)
identifier = ioids.get_child_element_text(ioids.parent_elem, 'org_id', 'identifier')
@@ -99,6 +102,16 @@ def do_import(self):
referenced_org = None
else:
referenced_org = None
+ if referenced_org:
+ owner = referenced_org.content_owner
+ if owner and owner != self.globals['cordaid']:
+ raise NotOwnedOrganisationException(
+ "Organisation {}, ID {}, is content owned by {}, ID {}. "
+ "Can't edit the data.".format(
+ referenced_org.name, referenced_org.pk,
+ owner.name, owner.pk
+ ))
+
org_fields = {}
org_fields['long_name'] = self.get_child_element_text(
self.parent_elem, 'name', 'long_name').strip()
@@ -106,12 +119,15 @@ def do_import(self):
org_fields['new_organisation_type'] = int(self.get_child_element_text(
self.parent_elem, 'iati_organisation_type', 'new_organisation_type', 22))
org_fields['iati_org_id'] = self.get_child_element_text(
- self.parent_elem, 'iati_org_id', 'iati_org_id')
+ self.parent_elem, 'iati_org_id', 'iati_org_id')
+ org_fields['description'] = self.get_child_element_text(
+ self.parent_elem, 'description', 'description')
# TODO: validate URL
org_fields['url'] = self.get_child_element_text(self.parent_elem, 'url', 'url')
+ org_fields['content_owner'] = self.globals['cordaid']
if referenced_org:
- organisation = Organisation.objects.get(pk=referenced_org.pk)
+ organisation = referenced_org
changes = self.update_organisation(organisation, org_fields)
created = False
else:
diff --git a/akvo/iati/imports/mappers/defaults.py b/akvo/iati/imports/mappers/defaults.py
index c376cfbe3e..e602a74ea5 100644
--- a/akvo/iati/imports/mappers/defaults.py
+++ b/akvo/iati/imports/mappers/defaults.py
@@ -242,5 +242,6 @@ def do_import(self):
"""
humanitarian = self.get_attrib(self.parent_elem, 'humanitarian', 'humanitarian', None)
- humanitarian = self.to_boolean(humanitarian)
+ if humanitarian:
+ humanitarian = self.to_boolean(humanitarian)
return self.update_project_field('humanitarian', humanitarian)
diff --git a/akvo/iati/imports/mappers/locations.py b/akvo/iati/imports/mappers/locations.py
index 19e17fef11..d89ff65a82 100644
--- a/akvo/iati/imports/mappers/locations.py
+++ b/akvo/iati/imports/mappers/locations.py
@@ -5,6 +5,7 @@
# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
from ....rsr.models.country import Country, RecipientCountry
+from ....rsr.models.iati_import_log import LOG_ENTRY_TYPE
from ....rsr.models.location import AdministrativeLocation, ProjectLocation
from ....rsr.models.region import RecipientRegion
@@ -87,7 +88,8 @@ def do_import(self):
try:
country = Country.objects.get(iso_code=country_code)
except ObjectDoesNotExist as e:
- self.add_log('administrative', 'country', str(e))
+ self.add_log('administrative', 'country', str(e),
+ LOG_ENTRY_TYPE.VALUE_NOT_SAVED)
loc, created = ProjectLocation.objects.get_or_create(
location_target=self.project,
diff --git a/akvo/rsr/migrations/0069_auto_20160517_1515.py b/akvo/rsr/migrations/0069_auto_20160517_1515.py
new file mode 100644
index 0000000000..897cb8c8d0
--- /dev/null
+++ b/akvo/rsr/migrations/0069_auto_20160517_1515.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import akvo.rsr.fields
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('rsr', '0068_iaticheck'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='project',
+ name='background',
+ field=akvo.rsr.fields.ValidXMLTextField(help_text='This should describe the geographical, political, environmental, social and/or cultural context of the project, and any related activities that have already taken place or are underway. For links and styling of the text, Markdown is supported.', verbose_name='background', blank=True),
+ preserve_default=True,
+ ),
+ migrations.AlterField(
+ model_name='project',
+ name='current_status',
+ field=akvo.rsr.fields.ValidXMLTextField(help_text='Describe the situation at the start of the project. For links and styling of the text, Markdown is supported.', verbose_name='baseline situation', blank=True),
+ preserve_default=True,
+ ),
+ migrations.AlterField(
+ model_name='project',
+ name='goals_overview',
+ field=akvo.rsr.fields.ValidXMLTextField(help_text='Provide a brief description of the overall project goals. For links and styling of the text, Markdown is supported.', verbose_name='goals overview', blank=True),
+ preserve_default=True,
+ ),
+ migrations.AlterField(
+ model_name='project',
+ name='project_plan',
+ field=akvo.rsr.fields.ValidXMLTextField(help_text='Detailed information about the implementation of the project: the what, how, who and when. For links and styling of the text, Markdown is supported.', verbose_name='project plan', blank=True),
+ preserve_default=True,
+ ),
+ migrations.AlterField(
+ model_name='project',
+ name='sustainability',
+ field=akvo.rsr.fields.ValidXMLTextField(help_text='Describe how you aim to guarantee sustainability of the project until 10 years after project implementation. Think about the institutional setting, capacity-building, a cost recovery plan, products used, feasible arrangements for operation and maintenance, anticipation of environmental impact and social integration. For links and styling of the text, Markdown is supported.', verbose_name='sustainability', blank=True),
+ preserve_default=True,
+ ),
+ migrations.AlterField(
+ model_name='project',
+ name='target_group',
+ field=akvo.rsr.fields.ProjectLimitedTextField(help_text='This should include information about the people, organisations or resources that are being impacted by this project. For links and styling of the text, Markdown is supported.', verbose_name='target group', blank=True),
+ preserve_default=True,
+ ),
+ migrations.AlterField(
+ model_name='projectdocument',
+ name='title',
+ field=akvo.rsr.fields.ValidXMLCharField(default='Untitled document', help_text='Enter the title of your document.', max_length=100, verbose_name='document title', blank=True),
+ preserve_default=True,
+ ),
+ migrations.AlterField(
+ model_name='projectlocation',
+ name='activity_description',
+ field=akvo.rsr.fields.ValidXMLCharField(help_text='A description that qualifies the activity taking place at the location. This should not duplicate information provided in the main activity description, and should typically be used to distinguish between activities at multiple locations within a single iati-activity record.', max_length=2000, verbose_name='activity description', blank=True),
+ preserve_default=True,
+ ),
+ migrations.AlterField(
+ model_name='projectlocation',
+ name='description',
+ field=akvo.rsr.fields.ValidXMLCharField(help_text='This provides free text space for providing an additional description, if needed, of the actual target of the activity. A description that qualifies the location, not the activity.', max_length=2000, verbose_name='location description', blank=True),
+ preserve_default=True,
+ ),
+ migrations.AlterField(
+ model_name='sector',
+ name='sector_code',
+ field=akvo.rsr.fields.ValidXMLCharField(help_text='Please select DAC-5 or DAC-3 as the sector vocabulary first, then this field will be populated with the corresponding codes. For other vocabularies, it is possible to fill in any code. See these lists for the DAC-5 and DAC-3 sector codes: DAC-5 sector codes and DAC-3 sector codes.', max_length=25, verbose_name='sector code', blank=True),
+ preserve_default=True,
+ ),
+ ]
diff --git a/akvo/rsr/models/country.py b/akvo/rsr/models/country.py
index 5c1518610c..8caeb6e960 100644
--- a/akvo/rsr/models/country.py
+++ b/akvo/rsr/models/country.py
@@ -69,7 +69,10 @@ class RecipientCountry(models.Model):
def __unicode__(self):
if self.country:
- country_unicode = self.iati_country().name
+ try:
+ country_unicode = self.iati_country().name
+ except (AttributeError, codelist_models.Country.DoesNotExist):
+ country_unicode = self.country
else:
country_unicode = u'%s' % _(u'No country specified')
diff --git a/akvo/rsr/models/iati_import_job.py b/akvo/rsr/models/iati_import_job.py
index f20ff8a415..3db502880f 100644
--- a/akvo/rsr/models/iati_import_job.py
+++ b/akvo/rsr/models/iati_import_job.py
@@ -421,7 +421,7 @@ class CordaidZipIatiImportJob(IatiImportJob):
the use of the ZIP archive to extract all information needed, both for the importing of
organisations and activities.
Note that self.iati_xml_file is used to hold the whole Cordaid ZIP archive, not just the
- activities XML. The XML is instead put in self._iati_xml_file in check_file()
+ activities XML. The XML is instead injected directly into self.parse_xml()
"""
IATI_XML_ACTIVITIES = 'iati-activities'
@@ -437,6 +437,7 @@ def import_organisations(self):
ORGANISATIONS_ROOT = 'Relations'
ORGANISATIONS_CHILDREN = 'object'
CORDAID_ORG_ID = 273
+
self.add_log(u'CordaidZip: Starting organisations import.', LOG_ENTRY_TYPE.INFORMATIONAL)
organisations_xml = self.get_xml_file(ORGANISATIONS_FILENAME)
if self.parse_xml(organisations_xml, ORGANISATIONS_ROOT, ORGANISATIONS_CHILDREN):
@@ -445,21 +446,29 @@ def import_organisations(self):
organisations = None
cordaid = Organisation.objects.get(pk=CORDAID_ORG_ID)
if organisations:
- for object in organisations.findall(ORGANISATIONS_CHILDREN):
+ created_count = 0
+ updated_count = 0
+ for element in organisations.findall(ORGANISATIONS_CHILDREN):
try:
- org_mapper = Organisations(self, object, None, {'cordaid': cordaid})
+ org_mapper = Organisations(self, element, None, {'cordaid': cordaid})
organisation, changes, created = org_mapper.do_import()
if created:
self.log_creation(organisation)
+ created_count += 1
else:
self.log_changes(organisation, changes)
+ if changes:
+ updated_count += 1
except Exception as e:
self.add_log(
u'CordaidZip: Critical error when importing '
u'organisations: {}'.format(e.message),
LOG_ENTRY_TYPE.CRITICAL_ERROR)
- self.add_log(u'CordaidZip: Organisations import done.', LOG_ENTRY_TYPE.INFORMATIONAL)
+ self.add_log(u'CordaidZip: Organisations import done. '
+ u'{} organisations created, {} organisations updated'.format(
+ created_count, updated_count),
+ LOG_ENTRY_TYPE.INFORMATIONAL)
def create_log_entry(self, organisation, action_flag=LOG_ENTRY_TYPE.ACTION_UPDATE,
change_message=u''):
@@ -507,13 +516,17 @@ def parse_xml(self, xml_file, root_tag='', children_tag=''):
:return: True if all's well, False if not
"""
try:
- parsed_xml = etree.parse(xml_file)
+ parser = etree.XMLParser()
+ for line in xml_file:
+ parser.feed(line)
+ if line.strip() == "".format(root_tag):
+ break
+ objects_root = parser.close()
except Exception as e:
self.add_log('Error parsing XML file. Error message:\n{}'.format(e.message),
LOG_ENTRY_TYPE.CRITICAL_ERROR)
return False
- objects_root = parsed_xml.getroot()
if objects_root.tag == root_tag:
self._objects_root = objects_root
self.add_log(
@@ -552,7 +565,8 @@ def set_activities(self):
:return: True or False indicating success or failure
"""
IATI_XML_ACTIVITY = 'iati-activity'
- if self.parse_xml(self._iati_xml_file, self.IATI_XML_ACTIVITIES, IATI_XML_ACTIVITY):
+
+ if self.parse_xml(self.get_activities_file(), self.IATI_XML_ACTIVITIES, IATI_XML_ACTIVITY):
self.activities = self._objects_root
return True
else:
@@ -569,7 +583,10 @@ def check_file(self):
if file:
# Since we're using self.iati_xml_file for the Cordaid ZIP archive, we assign the IATI
# XML to self._iati_xml_file
- self._iati_xml_file = file
+ # self._iati_xml_file = file
+ # The above line doesn't work for some reason. For reasons not understood, when assiging
+ # the file to self._iat_xml_file the file isn't readable any more when we access is in
+ # parse_xml(). Instead we inject the file directly in the call to parse_xml()
self.add_log('Using iati-activities.xml from the Cordaid ZIP: {}'.format(self.iati_xml_file),
LOG_ENTRY_TYPE.INFORMATIONAL)
return True
diff --git a/akvo/rsr/models/location.py b/akvo/rsr/models/location.py
index c5aa85d7af..a6cea1481d 100644
--- a/akvo/rsr/models/location.py
+++ b/akvo/rsr/models/location.py
@@ -99,13 +99,13 @@ class ProjectLocation(BaseLocation):
help_text=_(u'The human-readable name for the location.')
)
description = ValidXMLCharField(
- _(u'location description'), blank=True, max_length=255,
+ _(u'location description'), blank=True, max_length=2000,
help_text=_(u'This provides free text space for providing an additional description, if '
u'needed, of the actual target of the activity. A description that qualifies '
u'the location, not the activity.')
)
activity_description = ValidXMLCharField(
- _(u'activity description'), blank=True, max_length=255,
+ _(u'activity description'), blank=True, max_length=2000,
help_text=_(u'A description that qualifies the activity taking place at the location. '
u'This should not duplicate information provided in the main activity '
u'description, and should typically be used to distinguish between activities '