Multifile upload support

osgeo_importer/api.py

@@ -33,6 +33,9 @@ class UploadedLayerResource(ModelResource):
     configuration_options = DictField(attribute='configuration_options', null=True)
     fields = ListField(attribute='fields')
     status = CharField(attribute='status', readonly=True, null=True)
+    file_type = CharField(attribute='file_type', readonly=True)
+    file_name = CharField(attribute='file_name', readonly=True)
+    layer_name = CharField(attribute='layer_name', readonly=True)
 
     class Meta:
         queryset = UploadLayer.objects.all()
@@ -50,17 +53,15 @@ def get_object_list(self, request):
     def clean_configuration_options(self, request, obj, configuration_options):
         return configuration_options
 
-    def import_layer(self, request, **kwargs):
-        """
-        Imports a layer
+    def import_layer(self, request, pk=None, **kwargs):
+        """Imports a layer
         """
         self.method_check(request, allowed=['post'])
 
-        b = Bundle()
-        b.request = request
+        bundle = Bundle(request=request)
 
         try:
-            obj = self.obj_get(b, pk=kwargs.get('pk'))
+            obj = self.obj_get(bundle, pk=pk)
         except UploadLayer.DoesNotExist:
             raise ImmediateHttpResponse(response=http.HttpNotFound())
 
@@ -80,11 +81,11 @@ def import_layer(self, request, **kwargs):
         if not configuration_options:
             raise ImmediateHttpResponse(response=http.HttpBadRequest('Configuration options missing.'))
 
-        uploaded_file = obj.upload.uploadfile_set.first()
+        uploaded_file = obj.upload_file
         import_result = import_object.delay(uploaded_file.id, configuration_options=configuration_options)
 
         # query the db again for this object since it may have been updated during the import
-        obj = self.obj_get(b, pk=kwargs.get('pk'))
+        obj = self.obj_get(bundle, pk=pk)
         obj.task_id = import_result.id
         obj.save()
 
@@ -117,7 +118,7 @@ class UploadedDataResource(ModelResource):
     """
 
     user = ForeignKey(UserResource, 'user')
-    file_size = CharField(attribute='filesize', readonly=True)
+    file_size = CharField(attribute='filesize', readonly=True, null=True)
     layers = ToManyField(UploadedLayerResource, 'uploadlayer_set', full=True)
     file_url = CharField(attribute='file_url', readonly=True, null=True)
 

osgeo_importer/forms.py

@@ -1,9 +1,73 @@
+import os
 from django import forms
 from .models import UploadFile
+from .validators import validate_extension, validate_inspector_can_read, validate_shapefiles_have_all_parts
+from zipfile import is_zipfile, ZipFile
+import tempfile
+import logging
+import shutil
+logger = logging.getLogger(__name__)
 
 
-class UploadFileForm(forms.ModelForm):
+class UploadFileForm(forms.Form):
+    file = forms.FileField(widget=forms.ClearableFileInput(attrs={'multiple': True}))
 
     class Meta:
         model = UploadFile
         fields = ['file']
+
+    def clean(self):
+        cleaned_data = super(UploadFileForm, self).clean()
+        outputdir = tempfile.mkdtemp()
+        files = self.files.getlist('file')
+        validfiles = []
+
+        # Create list of all potentially valid files, exploding first level zip files
+        for file in files:
+            if not validate_extension(file.name):
+                self.add_error('file', 'Filetype not supported.')
+                continue
+
+            if is_zipfile(file):
+                with ZipFile(file) as zip:
+                    for zipname in zip.namelist():
+                        if not validate_extension(zipname):
+                            self.add_error('file', 'Filetype in zip not supported.')
+                            continue
+                        validfiles.append(zipname)
+            else:
+                validfiles.append(file.name)
+        # Make sure shapefiles have all their parts
+        if not validate_shapefiles_have_all_parts(validfiles):
+            self.add_error('file', 'Shapefiles must include .shp,.dbf,.shx,.prj')
+        # Unpack all zip files and create list of cleaned file objects
+        cleaned_files = []
+        for file in files:
+            if file.name in validfiles:
+                with open(os.path.join(outputdir, file.name), 'w') as outfile:
+                    for chunk in file.chunks():
+                        outfile.write(chunk)
+                cleaned_files.append(outfile)
+            elif is_zipfile(file):
+                with ZipFile(file) as zip:
+                    for zipfile in zip.namelist():
+                        if zipfile in validfiles:
+                            with zip.open(zipfile) as f:
+                                with open(os.path.join(outputdir, zipfile), 'w') as outfile:
+                                    shutil.copyfileobj(f, outfile)
+                                    cleaned_files.append(outfile)
+
+        # After moving files in place make sure they can be opened by inspector
+        inspected_files = []
+        for cleaned_file in cleaned_files:
+            cleaned_file_path = os.path.join(outputdir, cleaned_file.name)
+            if not validate_inspector_can_read(cleaned_file_path):
+                self.add_error(
+                    'file',
+                    'Inspector could not read file {} or file is empty'.format(cleaned_file_path)
+                )
+                continue
+            inspected_files.append(cleaned_file)
+
+        cleaned_data['file'] = inspected_files
+        return cleaned_data

osgeo_importer/handlers/__init__.py

@@ -10,7 +10,9 @@
                            'osgeo_importer.handlers.geoserver.GeoWebCacheHandler',
                            'osgeo_importer.handlers.geoserver.GeoServerBoundsHandler',
                            'osgeo_importer.handlers.geoserver.GenericSLDHandler',
-                           'osgeo_importer.handlers.geonode.GeoNodePublishHandler']
+                           'osgeo_importer.handlers.geonode.GeoNodePublishHandler',
+                           'osgeo_importer.handlers.geoserver.GeoServerStyleHandler',
+                           'osgeo_importer.handlers.geonode.GeoNodeMetadataHandler']
 
 IMPORT_HANDLERS = getattr(settings, 'IMPORT_HANDLERS', DEFAULT_IMPORT_HANDLERS)
 

osgeo_importer/handlers/geonode/__init__.py

@@ -2,7 +2,10 @@
 from osgeo_importer.models import UploadLayer
 from osgeo_importer.handlers import ImportHandlerMixin
 from osgeo_importer.handlers import ensure_can_run
+from osgeo_importer.importers import UPLOAD_DIR
 from geonode.geoserver.helpers import gs_slurp
+from geonode.layers.metadata import set_metadata
+from geonode.layers.utils import resolve_regions
 from django.contrib.auth import get_user_model
 from django.conf import settings
 from django import db
@@ -64,9 +67,58 @@ def handle(self, layer, layer_config, *args, **kwargs):
 
         if self.importer.upload_file and results['layers'][0]['status'] == 'created':
             matched_layer = Layer.objects.get(name=results['layers'][0]['name'])
-            upload_layer = UploadLayer.objects.get(upload=self.importer.upload_file.upload,
+            upload_layer = UploadLayer.objects.get(upload_file=self.importer.upload_file.pk,
                                                    index=layer_config.get('index'))
             upload_layer.layer = matched_layer
             upload_layer.save()
 
         return results
+
+
+class GeoNodeMetadataHandler(ImportHandlerMixin):
+    """Import uploaded XML
+    """
+
+    def can_run(self, layer, layer_config, *args, **kwargs):
+        """
+        Only run this handler if the layer is found in Geoserver and the layer's style is the generic style.
+        """
+        if not layer_config.get('metadata', None):
+            return False
+
+        return True
+
+    @ensure_can_run
+    def handle(self, layer, layer_config, *args, **kwargs):
+        """Update metadata from XML
+        """
+        geonode_layer = Layer.objects.get(name=layer)
+        path = os.path.join(UPLOAD_DIR, str(self.importer.upload_file.upload.id))
+        xmlfile = os.path.join(path, layer_config.get('metadata'))
+        geonode_layer.metadata_uploaded = True
+        identifier, vals, regions, keywords = set_metadata(open(xmlfile).read())
+
+        regions_resolved, regions_unresolved = resolve_regions(regions)
+        keywords.extend(regions_unresolved)
+
+        # set regions
+        regions_resolved = list(set(regions_resolved))
+        if regions:
+            if len(regions) > 0:
+                geonode_layer.regions.add(*regions_resolved)
+
+        # set taggit keywords
+        keywords = list(set(keywords))
+        geonode_layer.keywords.add(*keywords)
+
+        # set model properties
+        for (key, value) in vals.items():
+            if key == "spatial_representation_type":
+                # value = SpatialRepresentationType.objects.get(identifier=value)
+                pass
+            else:
+                setattr(geonode_layer, key, value)
+
+        geonode_layer.save()
+
+        return geonode_layer

osgeo_importer/handlers/geoserver/__init__.py

@@ -5,9 +5,12 @@
 from django import db
 from django.conf import settings
 from osgeo_importer.handlers import ImportHandlerMixin, GetModifiedFieldsMixin, ensure_can_run
-from geoserver.catalog import FailedRequestError
+from osgeo_importer.importers import UPLOAD_DIR
+from geoserver.catalog import FailedRequestError, ConflictingDataError
 from geonode.geoserver.helpers import gs_catalog
 from geoserver.support import DimensionInfo
+from osgeo_importer.utils import increment_filename
+
 
 logger = logging.getLogger(__name__)
 
@@ -331,3 +334,64 @@ def handle(self, layer, layer_config, *args, **kwargs):
         """
         self.layer.default_style = 'point'
         self.catalog.save(self.layer)
+
+
+class GeoServerStyleHandler(GeoserverHandlerMixin):
+    """Adds styles to GeoServer Layer
+    """
+    catalog = gs_catalog
+    catalog._cache.clear()
+    workspace = 'geonode'
+
+    def can_run(self, layer, layer_config, *args, **kwargs):
+        """
+        Returns true if the configuration has enough information to run the handler.
+        """
+        if not any([layer_config.get('default_style', None), layer_config.get('styles', None)]):
+            return False
+
+        return True
+
+    @ensure_can_run
+    def handle(self, layer, layer_config, *args, **kwargs):
+        """
+        Handler specific params:
+        "default_sld": SLD to load as default_sld
+        "slds": SLDS to add to layer
+        """
+        lyr = self.catalog.get_layer(layer)
+        path = os.path.join(UPLOAD_DIR, str(self.importer.upload_file.upload.id))
+        default_sld = layer_config.get('default_style', None)
+        slds = layer_config.get('styles', None)
+        all_slds = []
+        if default_sld is not None:
+            slds.append(default_sld)
+
+        all_slds = list(set(slds))
+        # all_slds = [CheckFile(x) for x in all_slds if x is not None]
+
+        styles = []
+        default_style = None
+        for sld in all_slds:
+            with open(os.path.join(path, sld)) as s:
+                n = 0
+                sldname = os.path.splitext(sld)[0]
+                while True:
+                    n += 1
+                    try:
+                        self.catalog.create_style(sldname, s.read(), overwrite=False, workspace=self.workspace)
+                    except ConflictingDataError:
+                        sldname = increment_filename(sldname)
+                    if n >= 100:
+                        break
+
+                style = self.catalog.get_style(sldname, workspace=self.workspace)
+                if sld == default_sld:
+                    default_style = style
+                styles.append(style)
+
+        lyr.styles = list(set(lyr.styles + styles))
+        if default_style is not None:
+            lyr.default_style = default_style
+        self.catalog.save(lyr)
+        return {'default_style': default_style.filename}

osgeo_importer/importers.py

@@ -17,13 +17,26 @@
 from django.conf import settings
 from django import db
 import logging
+from django.core.files.storage import FileSystemStorage
+
 
 logger = logging.getLogger(__name__)
 ogr.UseExceptions()
 
-
+MEDIA_ROOT = getattr(settings, 'MEDIA_ROOT', FileSystemStorage().location)
 OSGEO_IMPORTER = getattr(settings, 'OSGEO_IMPORTER', 'osgeo_importer.importers.OGRImport')
-RASTER_FILES = getattr(settings, 'RASTER_FILES', '/tmp')
+DEFAULT_SUPPORTED_EXTENSIONS = ['shp', 'shx', 'prj', 'dbf', 'kml', 'geojson', 'json',
+                                'tif', 'tiff', 'gpkg', 'csv', 'zip', 'xml', 'sld']
+VALID_EXTENSIONS = getattr(settings, 'OSGEO_IMPORTER_VALID_EXTENSIONS', DEFAULT_SUPPORTED_EXTENSIONS)
+
+RASTER_FILES = getattr(settings, 'OSGEO_IMPORTER_RASTER_FILES', os.path.join(MEDIA_ROOT, 'osgeo_importer_raster'))
+UPLOAD_DIR = getattr(settings, 'OSGEO_IMPORTER_UPLOAD_DIR', os.path.join(MEDIA_ROOT, 'osgeo_importer_uploads'))
+
+if not os.path.exists(RASTER_FILES):
+    os.makedirs(RASTER_FILES)
+
+if not os.path.exists(UPLOAD_DIR):
+    os.makedirs(UPLOAD_DIR)
 
 
 class Import(object):
@@ -37,8 +50,7 @@ class Import(object):
     enabled_handlers = IMPORT_HANDLERS
     source_inspectors = []
     target_inspectors = []
-    valid_extensions = ['gpx', 'geojson', 'json', 'zip', 'tar', 'kml', 'csv', 'shp',
-                        'tif', 'tiff', 'geotiff', 'gpkg']
+    valid_extensions = VALID_EXTENSIONS
 
     def filter_handler_results(self, handler_name):
         """

osgeo_importer/migrations/0003_uploadlayer_upload_file.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('osgeo_importer', '0002_auto_20160713_1429'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='uploadlayer',
+            name='upload_file',
+            field=models.ForeignKey(blank=True, to='osgeo_importer.UploadFile', null=True),
+        ),
+    ]

osgeo_importer/migrations/0004_uploadfile_file_type.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('osgeo_importer', '0003_uploadlayer_upload_file'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='uploadfile',
+            name='file_type',
+            field=models.CharField(max_length=50, null=True, blank=True),
+        ),
+    ]

osgeo_importer/migrations/0005_uploadlayer_layer_name.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('osgeo_importer', '0004_uploadfile_file_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='uploadlayer',
+            name='layer_name',
+            field=models.CharField(max_length=64, null=True),
+        ),
+    ]