[#1770] Models and basic procoess for IATI import

akvo/iati/iati_import.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+
+# Akvo RSR is covered by the GNU Affero General Public License.
+# See more details in the license.txt file located at the root folder of the Akvo RSR module.
+# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
+
+from ..rsr.models.iati_import_log import IatiImportLog
+
+
+class IatiImportProcess(object):
+    def check_file_or_url(self):
+        """
+        Check if the IATI import has a local_file or url specified.
+
+        If not, log in IatiImportLog and raise an error.
+        """
+        if not (self.iati_import.url or self.file):
+            IatiImportLog.objects.create(
+                iati_import=self.iati_import,
+                text=u'No file or URL specified',
+                error=True
+            )
+
+
+    def __init__(self, iati_import):
+        """
+        Initialise the IATI Import process.
+
+        :param iati_import: A IatiImport Django ORM object
+        """
+        self.iati_import = iati_import
+        self.file = iati_import.local_file
+        self.organisation = iati_import.reporting_organisation
+        self.user = iati_import.user
+
+        self.check_file_or_url()
+
+
+
+
+        # If local file doesn't exists, download the file locally first
+        if self.iati_import.url and not self.file:
+            try:
+                # Download file
+                iati_import.status = 2
+                iati_import.save()
+
+            except:
+                iati_import.status = 5
+                iati_import.save()
+
+        # Start the import only if a local file exists and the import has a reporting organisation
+        # and user specified
+        if iati_import.local_file and iati_import.reporting_organisation and iati_import.user:
+            try:
+                iati_import.status = 3
+                iati_import.save()
+
+                iati_import.status = 4
+                iati_import.save()
+                else:
+                    iati_import.status = 5
+                    iati_import.save()
+            except:
+                iati_import.status = 5
+                iati_import.save()
+
+

akvo/rsr/admin.py

@@ -1132,3 +1132,12 @@ def formfield_for_foreignkey(self, db_field, request=None, **kwargs):
         return super(EmploymentAdmin, self).formfield_for_foreignkey(db_field, request, **kwargs)
 
 admin.site.register(get_model('rsr', 'Employment'), EmploymentAdmin)
+
+
+class IatiImportAdmin(admin.ModelAdmin):
+    model = get_model('rsr', 'IatiImport')
+    list_display = ('__unicode__', 'reporting_organisation', 'status', 'start_date', 'end_date')
+    list_filter = ('status',)
+    search_fields = ('reporting_organisation__name', 'reporting_organisation__long_name',)
+
+admin.site.register(get_model('rsr', 'IatiImport'), IatiImportAdmin)

akvo/rsr/models/__init__.py

@@ -19,7 +19,7 @@
     create_publishing_status, create_organisation_account,
     create_payment_gateway_selector, donation_completed, act_on_log_entry,
     employment_post_save, employment_pre_save, update_project_budget,
-    update_project_funding, create_iati_file)
+    update_project_funding, create_iati_file, import_iati_file)
 
 from .benchmark import Benchmark, Benchmarkname
 from .budget_item import BudgetItem, BudgetItemLabel, CountryBudgetItem
@@ -32,6 +32,9 @@
 from .fss import Fss, FssForecast
 from .goal import Goal
 from .iati_export import IatiExport
+from .iati_import import IatiImport
+from .iati_import_log import IatiImportLog
+from .iati_project_import import IatiProjectImport
 from .indicator import Indicator, IndicatorPeriod
 from .invoice import Invoice
 from .internal_organisation_id import InternalOrganisationID
@@ -78,6 +81,9 @@
     'FssForecast',
     'Goal',
     'IatiExport',
+    'IatiImport',
+    'IatiImportLog',
+    'IatiProjectImport',
     'Indicator',
     'IndicatorPeriod',
     'Invoice',
@@ -316,3 +322,4 @@
 post_save.connect(create_api_key, sender=User)
 
 post_save.connect(create_iati_file, sender=IatiExport)
+post_save.connect(import_iati_file, sender=IatiImport)

akvo/rsr/models/iati_import.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+
+# Akvo RSR is covered by the GNU Affero General Public License.
+# See more details in the license.txt file located at the root folder of the Akvo RSR module.
+# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
+
+from django.db import models
+from django.conf import settings
+from django.utils.translation import ugettext_lazy as _
+
+from .iati_project_import import IatiProjectImport
+
+
+def file_path(self, filename):
+    return 'db/organisation/%s/iati_import/%s' % (str(self.reporting_organisation.pk), filename)
+
+
+STATUS_CODE = {
+    1: _(u'pending'),
+    2: _(u'retrieving file'),
+    3: _(u'import in progress'),
+    4: _(u'completed'),
+    5: _(u'cancelled')
+}
+
+
+class IatiImport(models.Model):
+    reporting_organisation = models.ForeignKey(
+        'Organisation', verbose_name=_(u'reporting organisation'), related_name='iati_imports'
+    )
+    # This user will be used for logging the actions in the LogEntry model
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL, verbose_name=_(u'user'), related_name='iati_imports'
+    )
+    url = models.URLField(_(u'url'), blank=True)
+    local_file = models.FileField(_(u'local file'), blank=True, upload_to=file_path)
+    status = models.PositiveSmallIntegerField(_(u'status'), default=1)
+    start_date = models.DateTimeField(_(u'start date'), null=True, blank=True)
+    end_date = models.DateTimeField(_(u'end date'), null=True, blank=True)
+    projects = models.ManyToManyField(
+        'Project', verbose_name=_(u'projects'), through=IatiProjectImport, blank=True
+    )
+
+    def __unicode__(self):
+        return u'%s %s (%s)' % (_(u'IATI import for'),
+                                self.reporting_organisation,
+                                self.show_status())
+
+    class Meta:
+        app_label = 'rsr'
+        verbose_name = _(u'IATI import')
+        verbose_name_plural = _(u'IATI imports')
+
+    def show_status(self):
+        if self.status not in STATUS_CODE.keys():
+            return _(u'unknown status')
+        else:
+            return STATUS_CODE[int(self.status)]

akvo/rsr/models/iati_import_log.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+# Akvo RSR is covered by the GNU Affero General Public License.
+# See more details in the license.txt file located at the root folder of the Akvo RSR module.
+# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from ..fields import ValidXMLTextField
+
+
+class IatiImportLog(models.Model):
+    iati_import = models.ForeignKey(
+        'IatiImport', verbose_name=_(u'iati_import'), related_name='iati_import_logs'
+    )
+    text = ValidXMLTextField(_(u'text'))
+    error = models.BooleanField(_(u'error'), default=False)
+    project = models.ForeignKey(
+        'Project', verbose_name=_(u'project'), related_name='iati_project_import_logs',
+        blank=True, null=True
+    )
+
+    def __unicode__(self):
+        return u'%s (ID: %s): %s' % (_(u'IATI import log'), str(self.iati_import.pk), self.text)
+
+    class Meta:
+        app_label = 'rsr'
+        verbose_name = _(u'IATI import log')
+        verbose_name_plural = _(u'IATI import logs')

akvo/rsr/models/iati_project_import.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+
+# Akvo RSR is covered by the GNU Affero General Public License.
+# See more details in the license.txt file located at the root folder of the Akvo RSR module.
+# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+
+ACTION_CODE = {
+    1: _(u'create'),
+    2: _(u'update')
+}
+
+
+STATUS_CODE = {
+    1: _(u'pending'),
+    2: _(u'import in progress'),
+    3: _(u'completed'),
+    4: _(u'cancelled')
+}
+
+
+class IatiProjectImport(models.Model):
+    iati_import = models.ForeignKey(
+        'IatiImport', verbose_name=_(u'iati_import'), related_name='iati_project_imports'
+    )
+    project = models.ForeignKey(
+        'Project', verbose_name=_(u'project'), related_name='iati_project_imports'
+    )
+    action = models.PositiveSmallIntegerField(_(u'action'))
+    status = models.PositiveSmallIntegerField(_(u'status'), default=1)
+    start_date = models.DateTimeField(_(u'start date'), null=True, blank=True)
+    end_date = models.DateTimeField(_(u'end date'), null=True, blank=True)
+    errors = models.BooleanField(_(u'errors'), default=False)
+
+    def __unicode__(self):
+        return u'%s %s' % (_(u'IATI project import for'), self.project)
+
+    class Meta:
+        app_label = 'rsr'
+        verbose_name = _(u'IATI project import')
+        verbose_name_plural = _(u'IATI project imports')
+
+    def show_action(self):
+        if self.action not in ACTION_CODE.keys():
+            return _(u'unknown action')
+        else:
+            return ACTION_CODE[int(self.action)]
+
+    def show_status(self):
+        if self.status not in STATUS_CODE.keys():
+            return _(u'unknown status')
+        else:
+            return STATUS_CODE[int(self.status)]

akvo/rsr/signals.py

@@ -24,6 +24,7 @@
 
 from akvo.utils import send_donation_confirmation_emails, rsr_send_mail, rsr_send_mail_to_users
 from akvo.iati.iati_export import IatiXML
+from akvo.iati.iati_import import IatiImportProcess
 
 
 def create_publishing_status(sender, **kwargs):
@@ -308,3 +309,23 @@ def create_iati_file(sender, **kwargs):
             iati_export.status = 4
             iati_export.save()
         post_save.connect(create_iati_file, sender=sender)
+
+
+def import_iati_file(sender, **kwargs):
+    """
+    Import an IATI XML file when an entry in the iati_import table is saved.
+
+    :param sender: IatiImport model
+    """
+    ## TODO: only fire when created
+    iati_import = kwargs.get("instance", None)
+
+    if iati_import:
+        post_save.disconnect(import_iati_file, sender=sender)
+        try:
+            IatiImportProcess(iati_import)
+        except:
+            iati_import.status = 5
+            iati_import.errors = True
+            iati_import.save()
+        post_save.connect(import_iati_file, sender=sender)