From c2edf90865f27009b85a2ccc5387220b6ac25ce9 Mon Sep 17 00:00:00 2001 From: "david.w.bitner" Date: Wed, 17 Aug 2016 12:05:10 -0500 Subject: [PATCH 01/27] use form for multifile uploads --- osgeo_importer/forms.py | 74 ++++++++++++++++++++++++++++++++++++++++- osgeo_importer/tests.py | 4 ++- osgeo_importer/views.py | 64 +++++++++++++++++------------------ 3 files changed, 108 insertions(+), 34 deletions(-) diff --git a/osgeo_importer/forms.py b/osgeo_importer/forms.py index 05aecbb..40f340b 100644 --- a/osgeo_importer/forms.py +++ b/osgeo_importer/forms.py @@ -1,9 +1,81 @@ +import os from django import forms from .models import UploadFile +import zipfile +import tempfile -class UploadFileForm(forms.ModelForm): +class XXXUploadFileForm(forms.ModelForm): class Meta: model = UploadFile fields = ['file'] + +class UploadFileForm(forms.Form): + file = forms.FileField(widget=forms.ClearableFileInput(attrs={'multiple': True})) + + def validate_extension(filename): + base, extension = os.path.splitext(filename) + extension = extension.lstrip('.').lower() + if extension not in OSGEO_IMPORTER_VALID_EXTENSIONS: + raise forms.ValidationError("%s files not accepted by importer. file: %s" % (extension, filename)) + + def validate_shapefile_has_all_parts(self,validfiles): + shp = [] + prj = [] + dbf = [] + shx = [] + for file in validfiles: + base, extension = os.path.splitext(filename) + extension = extension.lstrip('.').lower() + if extension == 'shp': + shp.push(base) + elif extension == 'prj': + prj.push(base) + elif extension == 'dbf': + dbf.push(base) + elif extension == 'shx': + shx.push(base) + if set(shp) == set(prj) == set(dbf) == set(shx): + return True + else: + raise forms.ValidationError("All Shapefiles must include .shp, .prj, .dbf, and .shx") + + def clean(self): + cleaned_data = super(UploadFileForm, self).clean() + outputdir = tempfile.mkdtemp() + files = self.files.getlist('files') + validfiles = [] + + # Create list of all potentially valid files, exploding first level zip files + for file in files: + self.validate_extension(file_ext) + + if zipfile.is_zipfile(file): + with zipfile.ZipFile(file) as zip: + for zipfile in zip.namelist(): + zipfile_base, zipfile_ext = os.path.splitext(zipfile) + self.validate_extension(zipfile_ext) + validfiles.push(zipfile) + else: + validfiles.push(file.name) + + # Make sure shapefiles have all their parts + self.validate_shapefile_has_all_parts(validfiles) + + # Unpack all zip files and create list of cleaned file objects + cleaned_files = [] + for file in files: + if file in validfiles: + # with open(os.path.join(outputdir,file.name),'w') as outfile: + # outfile.write(file.read()) + cleaned_files.push(file) + elif zipfile.is_zipfile(file): + with zipfile.ZipFile(file) as zip: + for zipfile in zip.namelist(): + if zipfile in validfiles: + with zip.open(zipfile) as f: + with open(os.path.join(outputdir,zipfile)) as outfile: + shutil.copyfileobj(f,outfile) + cleaned_files.push(outfile) + cleaned_data['files'] = cleaned_files diff --git a/osgeo_importer/tests.py b/osgeo_importer/tests.py index 02ecea1..27c2251 100644 --- a/osgeo_importer/tests.py +++ b/osgeo_importer/tests.py @@ -22,7 +22,7 @@ from geonode.geoserver.helpers import ogc_server_settings from osgeo_importer.models import UploadLayer from osgeo_importer.models import validate_file_extension, ValidationError, validate_inspector_can_read -from osgeo_importer.models import UploadedData +from osgeo_importer.models import UploadedData, UploadFile from osgeo_importer.handlers.geoserver import GeoWebCacheHandler from osgeo_importer.importers import OSGEO_IMPORTER, OGRImport @@ -776,6 +776,8 @@ def naming_an_import(self): with open(f) as fp: response = c.post(reverse('uploads-new'), {'file': fp}, follow=True) + print UploadFile.objects.first() + payload = {'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', diff --git a/osgeo_importer/views.py b/osgeo_importer/views.py index e6dc40f..d64b728 100644 --- a/osgeo_importer/views.py +++ b/osgeo_importer/views.py @@ -1,3 +1,4 @@ +import os import json import logging from django.http import HttpResponse @@ -8,6 +9,7 @@ from .importers import OSGEO_IMPORTER from .inspectors import OSGEO_INSPECTOR from .utils import import_string +from django.conf import settings OSGEO_INSPECTOR = import_string(OSGEO_INSPECTOR) OSGEO_IMPORTER = import_string(OSGEO_IMPORTER) @@ -76,40 +78,38 @@ class FileAddView(FormView, ImportHelper, JSONResponseMixin): template_name = 'osgeo_importer/new.html' json = False - def create_upload_session(self, upload_file): - """ - Creates an upload session from the file. - """ - upload = UploadedData.objects.create(user=self.request.user, state='UPLOADED', complete=True) - upload_file.upload = upload - upload_file.save() - upload.size = upload_file.file.size - upload.name = upload_file.name - upload.file_type = self.get_file_type(upload_file.file.path) - upload.save() - - description = self.get_fields(upload_file.file.path) - if not description: - logger.debug("No layers detected; assuming raster") - configuration_options = DEFAULT_LAYER_CONFIGURATION.copy() - upload.uploadlayer_set.add(UploadLayer(name=upload.name, - configuration_options=configuration_options)) - - for layer in description: - configuration_options = DEFAULT_LAYER_CONFIGURATION.copy() - configuration_options.update({'index': layer.get('index')}) - upload.uploadlayer_set.add(UploadLayer(name=layer.get('layer_name'), - fields=layer.get('fields', {}), - index=layer.get('index'), - feature_count=layer.get('feature_count'), - configuration_options=configuration_options)) - upload.save() - return upload - def form_valid(self, form): + upload = UploadedData(user=self.request.user) + upload.save() - form.save(commit=True) - upload = self.create_upload_session(form.instance) + # Create Upload Directory based on Upload PK + outpath = os.path.join('/uploads',str(upload.pk)) + outdir = os.path.join(settings.MEDIA_ROOT,outpath) + + # Move all files to uploads directory using upload pk + # Must be done for all files before saving upfile for validation + for each in form.cleaned_data['files']: + shutil.move(each.path, os.path.join(outpath,each.name)) + + # Loop through and create uploadfiles and uploadlayers + for each in form.cleaned_data['files']: + upfile = UploadFile(upload=upload) + upfile.file.name = os.path.join(outpath,each.name) + upfile.save() + + upfile_base, upfile_ext = os.path.splitext(each.name) + if upfile_ext.lower() not in ['.xml','.sld','.prj','.dbf','shx']: + description = self.get_fields(upload_file.file.path) + for layer in description: + configuration_options = DEFAULT_LAYER_CONFIGURATION.copy() + configuration_options.update({'index': layer.get('index')}) + upload.uploadlayer_set.add(UploadLayer(name=layer.get('name'), + fields=layer.get('fields', {}), + index=layer.get('index'), + feature_count=layer.get('feature_count',None), + configuration_options=configuration_options)) + upload.complete = True + upload.state = 'UPLOADED' if self.json: return self.render_to_json_response({'state': upload.state, 'id': upload.id}) From 863d5f0eaa8cf851fc15520b060a558661cd2847 Mon Sep 17 00:00:00 2001 From: "david.w.bitner" Date: Thu, 25 Aug 2016 15:26:21 -0500 Subject: [PATCH 02/27] sld config working --- osgeo_importer/forms.py | 91 ++++++++----- osgeo_importer/handlers/__init__.py | 3 +- osgeo_importer/handlers/geoserver/__init__.py | 67 +++++++++- osgeo_importer/models.py | 1 + osgeo_importer/tests.py | 121 +++++++++++++++++- osgeo_importer/views.py | 47 ++++--- osgeo_importer_prj/settings.py | 1 + 7 files changed, 280 insertions(+), 51 deletions(-) diff --git a/osgeo_importer/forms.py b/osgeo_importer/forms.py index 40f340b..f184701 100644 --- a/osgeo_importer/forms.py +++ b/osgeo_importer/forms.py @@ -1,9 +1,15 @@ import os from django import forms from .models import UploadFile -import zipfile +from .utils import NoDataSourceFound, load_handler +from .importers import OSGEO_IMPORTER +from zipfile import is_zipfile, ZipFile import tempfile +import logging +import shutil +from django.conf import settings +logger = logging.getLogger(__name__) class XXXUploadFileForm(forms.ModelForm): @@ -13,12 +19,17 @@ class Meta: class UploadFileForm(forms.Form): file = forms.FileField(widget=forms.ClearableFileInput(attrs={'multiple': True})) - - def validate_extension(filename): + + class Meta: + model = UploadFile + fields = ['file'] + + def validate_extension(self,filename): base, extension = os.path.splitext(filename) extension = extension.lstrip('.').lower() - if extension not in OSGEO_IMPORTER_VALID_EXTENSIONS: - raise forms.ValidationError("%s files not accepted by importer. file: %s" % (extension, filename)) + if extension not in settings.OSGEO_IMPORTER_VALID_EXTENSIONS: + logger.debug('Validation Error: %s files not accepted by importer. file: %s',extension,filename) + self.add_error('file',"%s files not accepted by importer. file: %s" % (extension, filename)) def validate_shapefile_has_all_parts(self,validfiles): shp = [] @@ -26,56 +37,76 @@ def validate_shapefile_has_all_parts(self,validfiles): dbf = [] shx = [] for file in validfiles: - base, extension = os.path.splitext(filename) + base, extension = os.path.splitext(file) extension = extension.lstrip('.').lower() if extension == 'shp': - shp.push(base) + shp.append(base) elif extension == 'prj': - prj.push(base) + prj.append(base) elif extension == 'dbf': - dbf.push(base) + dbf.append(base) elif extension == 'shx': - shx.push(base) + shx.append(base) if set(shp) == set(prj) == set(dbf) == set(shx): return True else: - raise forms.ValidationError("All Shapefiles must include .shp, .prj, .dbf, and .shx") + logger.debug('Validation Error: All Shapefiles must include .shp, .prj, .dbf, and .shx') + self.add_error('file',"All Shapefiles must include .shp, .prj, .dbf, and .shx") + + def validate_inspector_can_read(self,filename): + try: + importer = load_handler(OSGEO_IMPORTER, filename) + data, inspector = importer.open_source_datastore(filename) + except NoDataSourceFound: + self.add_error('file','Unable to open file: %s' % filename) def clean(self): + logger.debug('..Cleaning...') cleaned_data = super(UploadFileForm, self).clean() outputdir = tempfile.mkdtemp() - files = self.files.getlist('files') + files = self.files.getlist('file') validfiles = [] # Create list of all potentially valid files, exploding first level zip files for file in files: - self.validate_extension(file_ext) + logger.debug('cleaning %s',file.name) + self.validate_extension(file.name) - if zipfile.is_zipfile(file): - with zipfile.ZipFile(file) as zip: - for zipfile in zip.namelist(): - zipfile_base, zipfile_ext = os.path.splitext(zipfile) - self.validate_extension(zipfile_ext) - validfiles.push(zipfile) + if is_zipfile(file): + with ZipFile(file) as zip: + for zipname in zip.namelist(): + logger.debug('Checking extensions in zipfile %s file %s',file,zipname) + self.validate_extension(zipname) + validfiles.append(zipname) else: - validfiles.push(file.name) + validfiles.append(file.name) # Make sure shapefiles have all their parts self.validate_shapefile_has_all_parts(validfiles) - + logger.debug('valid files found: %s',validfiles) # Unpack all zip files and create list of cleaned file objects cleaned_files = [] for file in files: - if file in validfiles: - # with open(os.path.join(outputdir,file.name),'w') as outfile: - # outfile.write(file.read()) - cleaned_files.push(file) - elif zipfile.is_zipfile(file): - with zipfile.ZipFile(file) as zip: + logger.debug('file: %s',file) + if file.name in validfiles: + with open(os.path.join(outputdir,file.name),'w') as outfile: + for chunk in file.chunks(): + outfile.write(chunk) + cleaned_files.append(outfile) + elif is_zipfile(file): + with ZipFile(file) as zip: for zipfile in zip.namelist(): if zipfile in validfiles: with zip.open(zipfile) as f: - with open(os.path.join(outputdir,zipfile)) as outfile: + with open(os.path.join(outputdir,zipfile),'w') as outfile: shutil.copyfileobj(f,outfile) - cleaned_files.push(outfile) - cleaned_data['files'] = cleaned_files + cleaned_files.append(outfile) + + # After moving files in place make sure they can be opened by inspector + for cleaned_file in cleaned_files: + logger.debug('About to inspect %s in form',os.path.join(outputdir,file.name)) + self.validate_inspector_can_read(os.path.join(outputdir,file.name)) + + logger.debug('cleaned_files %s',cleaned_files) + cleaned_data['file'] = cleaned_files + return cleaned_data diff --git a/osgeo_importer/handlers/__init__.py b/osgeo_importer/handlers/__init__.py index b2d9f80..35f3c78 100644 --- a/osgeo_importer/handlers/__init__.py +++ b/osgeo_importer/handlers/__init__.py @@ -10,7 +10,8 @@ 'osgeo_importer.handlers.geoserver.GeoWebCacheHandler', 'osgeo_importer.handlers.geoserver.GeoServerBoundsHandler', 'osgeo_importer.handlers.geoserver.GenericSLDHandler', - 'osgeo_importer.handlers.geonode.GeoNodePublishHandler'] + 'osgeo_importer.handlers.geonode.GeoNodePublishHandler', + 'osgeo_importer.handlers.geoserver.GeoServerStyleHandler'] IMPORT_HANDLERS = getattr(settings, 'IMPORT_HANDLERS', DEFAULT_IMPORT_HANDLERS) diff --git a/osgeo_importer/handlers/geoserver/__init__.py b/osgeo_importer/handlers/geoserver/__init__.py index a402970..8beb9a4 100644 --- a/osgeo_importer/handlers/geoserver/__init__.py +++ b/osgeo_importer/handlers/geoserver/__init__.py @@ -5,9 +5,12 @@ from django import db from django.conf import settings from osgeo_importer.handlers import ImportHandlerMixin, GetModifiedFieldsMixin, ensure_can_run -from geoserver.catalog import FailedRequestError +from geoserver.catalog import FailedRequestError, ConflictingDataError from geonode.geoserver.helpers import gs_catalog from geoserver.support import DimensionInfo +from django.core.files.storage import FileSystemStorage +from osgeo_importer.utils import increment_filename + logger = logging.getLogger(__name__) @@ -331,3 +334,65 @@ def handle(self, layer, layer_config, *args, **kwargs): """ self.layer.default_style = 'point' self.catalog.save(self.layer) + + +class GeoServerStyleHandler(GeoserverHandlerMixin): + """ + Adds styles to GeoServer Layer + """ + catalog = gs_catalog + catalog._cache.clear() + workspace = 'geonode' + + def can_run(self, layer, layer_config, *args, **kwargs): + """ + Returns true if the configuration has enough information to run the handler. + """ + if not any([layer_config.get('default_style', None), layer_config.get('styles', None)]): + return False + + return True + + @ensure_can_run + def handle(self, layer, layer_config, *args, **kwargs): + """ + Handler specific params: + "default_sld": SLD to load as default_sld + "slds": SLDS to add to layer + """ + lyr = self.catalog.get_layer(layer) + path = os.path.join(FileSystemStorage().location,'osgeo_importer_uploads', str(self.importer.upload_file.upload.id)) + default_sld = layer_config.get('default_style', None) + slds = layer_config.get('styles', None) + all_slds = [] + if default_sld is not None: + slds.append(default_sld) + + all_slds = list(set(slds)) + # all_slds = [CheckFile(x) for x in all_slds if x is not None] + + styles = [] + default_style = None + for sld in all_slds: + with open(os.path.join(path, sld)) as s: + n = 0 + sldname = os.path.splitext(sld)[0] + while True: + n += 1 + try: + self.catalog.create_style(sldname, s.read(), overwrite=False, workspace=self.workspace) + except ConflictingDataError: + sldname = increment_filename(sldname) + if n >= 100: + break + + style = self.catalog.get_style(sldname, workspace=self.workspace) + if sld == default_sld: + default_style = style + styles.append(style) + + lyr.styles = list(set(lyr.styles + styles)) + if default_style is not None: + lyr.default_style = default_style + self.catalog.save(lyr) + return {'default_style': default_style.filename} diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index c2186e2..0359c38 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -79,6 +79,7 @@ def validate_inspector_can_read(value): """ Validates Geospatial data. """ + logger.debug('Inspecting Can Read %s',value.name) name = value.name root, extension = os.path.splitext(name.lower()) diff --git a/osgeo_importer/tests.py b/osgeo_importer/tests.py index 27c2251..490fac5 100644 --- a/osgeo_importer/tests.py +++ b/osgeo_importer/tests.py @@ -147,6 +147,57 @@ def generic_import(self, file, configuration_options=[{'index': 0}]): return layer_results[0] + def generic_api_upload(self, files, configuration_options=None): + """ + Tests the import api. + """ + c = AdminClient() + c.login_as_non_admin() + + # Upload Files + if isinstance(files, type(str())): + files = [files] + outfiles = [] + handles = {} + for file in files: + f = os.path.join( + os.path.dirname(__file__), + '..', + 'importer-test-files', + file) + handles[file] = open(f) + outfiles.append(SimpleUploadedFile(file, handles[file].read())) + response = c.post( + reverse('uploads-new-json'), + {'file': outfiles, + 'json': json.dumps(configuration_options)}, + follow=True) + # Clean up file handles + for file, handle in handles.items(): + handle.close() + content = json.loads(response.content) + self.assertEqual(response.status_code, 200) + self.assertEqual(content['id'], 1) + + # Configure Uploaded Files + uploadid = content['id'] + upload_layers = UploadLayer.objects.filter(upload_id=uploadid) + + for upload_layer in upload_layers: + print upload_layer.id + print upload_layer.name + for config in configuration_options: + if config['upload_file_name'] == upload_layer.name: + payload = config['config'] + response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload_layer.id), data=json.dumps(payload), + content_type='application/json') + self.assertTrue(response.status_code, 200) + response = c.get('/importer-api/data-layers/{0}/'.format(upload_layer.id), + content_type='application/json') + self.assertTrue(response.status_code, 200) + + return content + def generic_raster_import(self, file, configuration_options=[{'index': 0}]): f = file filename = os.path.join(os.path.dirname(__file__), '..', 'importer-test-files', f) @@ -160,6 +211,70 @@ def generic_raster_import(self, file, configuration_options=[{'index': 0}]): self.assertTrue(l.GetDriver().ShortName, 'GTiff') return layer + def test_multi_upload(self): + """ + Tests Uploading Multiple Files + """ + upload = self.generic_api_upload( + ['boxes_with_year_field.zip', + 'boxes_with_date.zip', + 'point_with_date.geojson'], + [{'upload_file_name': 'boxes_with_year_field.shp', + 'config': [{'index': 0}]}, + {'upload_file_name': 'boxes_with_date.shp', + 'config': [{'index': 0}]}, + {'upload_file_name': 'point_with_date.geojson', + 'config': [{'index': 0}]} + ] + ) + self.assertEqual(9, upload['count']) + + def test_upload_with_slds(self): + """ + Tests Uploading sld + """ + upload = self.generic_api_upload( + ['boxes_with_date.zip', + 'boxes.sld', + 'boxes1.sld'], + [{'upload_file_name': 'boxes_with_date.shp', + 'config': [{'index': 0, 'default_style': 'boxes.sld', + 'styles': ['boxes.sld', 'boxes1.sld']}]} + ] + ) + self.assertEqual(6, upload['count']) + uploadid = upload['id'] + uploadobj = UploadedData.objects.get(pk=uploadid) + uplayers = UploadLayer.objects.filter(upload=uploadid) + layerid = uplayers[0].pk + + upfiles_cnt = UploadFile.objects.filter(upload=uploadid).count() + self.assertEqual(6,upfiles_cnt) + + layer = Layer.objects.get(pk=layerid) + gslayer = self.cat.get_layer(layer.name) + default_style = gslayer.default_style + self.cat._cache.clear() + self.assertEqual('boxes.sld',default_style.filename) + + def test_upload_with_metadata(self): + """ + Tests Uploading metadata + """ + upload = self.generic_api_upload( + ['boxes_with_date.zip', + 'samplemetadata.xml',], + [{'upload_file_name': 'boxes_with_date.shp', + 'config': [{'index': 0, 'metadata': 'samplemetadata.xml'}]} + ] + ) + self.assertEqual(5, upload['count']) + print upload + layerid = upload['uploaded'][0]['pk']; + layer = Layer.objects.get(pk=layerid) + self.assertEqual(layer.language, 'eng') + self.assertEqual(layer.title, 'Old_Americas_LSIB_Polygons_Detailed_2013Mar') + def test_raster(self): """ Tests raster import @@ -505,10 +620,10 @@ def test_file_add_view(self): self.assertTrue(len(response.context['object_list']) == 1) upload = response.context['object_list'][0] self.assertEqual(upload.user.username, 'non_admin') - self.assertEqual(upload.file_type, 'GeoJSON') + # self.assertEqual(upload.file_type, 'GeoJSON') self.assertTrue(upload.uploadlayer_set.all()) self.assertEqual(upload.state, 'UPLOADED') - self.assertIsNotNone(upload.name) + # self.assertIsNotNone(upload.name) uploaded_file = upload.uploadfile_set.first() self.assertTrue(os.path.exists(uploaded_file.file.path)) @@ -776,8 +891,6 @@ def naming_an_import(self): with open(f) as fp: response = c.post(reverse('uploads-new'), {'file': fp}, follow=True) - print UploadFile.objects.first() - payload = {'index': 0, 'convert_to_date': ['date'], 'start_date': 'date', diff --git a/osgeo_importer/views.py b/osgeo_importer/views.py index d64b728..57d26f9 100644 --- a/osgeo_importer/views.py +++ b/osgeo_importer/views.py @@ -1,20 +1,27 @@ import os import json import logging +import shutil from django.http import HttpResponse from django.views.generic import FormView, ListView, TemplateView from django.core.urlresolvers import reverse_lazy from .forms import UploadFileForm -from .models import UploadedData, UploadLayer, DEFAULT_LAYER_CONFIGURATION +from .models import UploadedData, UploadLayer, UploadFile, DEFAULT_LAYER_CONFIGURATION from .importers import OSGEO_IMPORTER from .inspectors import OSGEO_INSPECTOR from .utils import import_string from django.conf import settings +from django.core.files.storage import FileSystemStorage OSGEO_INSPECTOR = import_string(OSGEO_INSPECTOR) OSGEO_IMPORTER = import_string(OSGEO_IMPORTER) logger = logging.getLogger(__name__) +if logger.handlers: + for handler in logger.handlers: + logger.removeHandler(handler) +logging.basicConfig(filename='/vagrant/views.log', level=logging.DEBUG) + class JSONResponseMixin(object): @@ -79,40 +86,50 @@ class FileAddView(FormView, ImportHelper, JSONResponseMixin): json = False def form_valid(self, form): - upload = UploadedData(user=self.request.user) + upload = UploadedData.objects.create(user=self.request.user) upload.save() # Create Upload Directory based on Upload PK - outpath = os.path.join('/uploads',str(upload.pk)) - outdir = os.path.join(settings.MEDIA_ROOT,outpath) + outpath = os.path.join('osgeo_importer_uploads',str(upload.pk)) + outdir = os.path.join(FileSystemStorage().location,outpath) + if not os.path.exists(outdir): + os.makedirs(outdir) + logger.debug("%s",outdir) + logger.debug("cleaned_data: %s",form.cleaned_data) # Move all files to uploads directory using upload pk # Must be done for all files before saving upfile for validation - for each in form.cleaned_data['files']: - shutil.move(each.path, os.path.join(outpath,each.name)) + finalfiles = [] + for each in form.cleaned_data['file']: + tofile = os.path.join(outdir,os.path.basename(each.name)) + logger.debug('moving %s to %s', each.name, tofile) + shutil.move(each.name, tofile) + finalfiles.append(tofile) # Loop through and create uploadfiles and uploadlayers - for each in form.cleaned_data['files']: - upfile = UploadFile(upload=upload) - upfile.file.name = os.path.join(outpath,each.name) + uplayers=[] + for each in finalfiles: + upfile = UploadFile.objects.create(upload=upload) + upfile.file.name = each upfile.save() - - upfile_base, upfile_ext = os.path.splitext(each.name) - if upfile_ext.lower() not in ['.xml','.sld','.prj','.dbf','shx']: - description = self.get_fields(upload_file.file.path) + upfile_basename = os.path.basename(each) + upfile_root, upfile_ext = os.path.splitext(upfile_basename) + if upfile_ext.lower() not in ['.prj','.dbf','.shx']: + description = self.get_fields(each) for layer in description: configuration_options = DEFAULT_LAYER_CONFIGURATION.copy() configuration_options.update({'index': layer.get('index')}) - upload.uploadlayer_set.add(UploadLayer(name=layer.get('name'), + upload.uploadlayer_set.add(UploadLayer(name=upfile_basename, fields=layer.get('fields', {}), index=layer.get('index'), feature_count=layer.get('feature_count',None), configuration_options=configuration_options)) upload.complete = True upload.state = 'UPLOADED' + upload.save() if self.json: - return self.render_to_json_response({'state': upload.state, 'id': upload.id}) + return self.render_to_json_response({'state': upload.state, 'id': upload.id, 'count': UploadFile.objects.filter(upload=upload.id).count()}) return super(FileAddView, self).form_valid(form) diff --git a/osgeo_importer_prj/settings.py b/osgeo_importer_prj/settings.py index 4da0405..140c1f7 100644 --- a/osgeo_importer_prj/settings.py +++ b/osgeo_importer_prj/settings.py @@ -80,5 +80,6 @@ OSGEO_DATASTORE = 'datastore' OSGEO_IMPORTER_GEONODE_ENABLED = True +OSGEO_IMPORTER_VALID_EXTENSIONS = ['shp','shx','prj','dbf','kml','geojson','json','tif','tiff','gpkg','csv','zip','xml','sld'] LOGGING['loggers']['osgeo_importer'] = {"handlers": ["console"], "level": "DEBUG"} DATABASE_ROUTERS = ['osgeo_importer_prj.dbrouters.DefaultOnlyMigrations'] From c1cc19a9649953e3a7854923484705b3be5a6617 Mon Sep 17 00:00:00 2001 From: "david.w.bitner" Date: Fri, 26 Aug 2016 16:47:51 -0500 Subject: [PATCH 03/27] move validators to validators.py get tests working --- osgeo_importer/forms.py | 67 +++++-------------- osgeo_importer/handlers/__init__.py | 3 +- osgeo_importer/handlers/geonode/__init__.py | 53 +++++++++++++++ osgeo_importer/handlers/geoserver/__init__.py | 3 +- osgeo_importer/importers.py | 18 +++-- osgeo_importer/validators.py | 62 +++++++++++++++++ 6 files changed, 151 insertions(+), 55 deletions(-) create mode 100644 osgeo_importer/validators.py diff --git a/osgeo_importer/forms.py b/osgeo_importer/forms.py index f184701..b95740b 100644 --- a/osgeo_importer/forms.py +++ b/osgeo_importer/forms.py @@ -3,6 +3,7 @@ from .models import UploadFile from .utils import NoDataSourceFound, load_handler from .importers import OSGEO_IMPORTER +from .validators import validate_extension, validate_inspector_can_read, validate_shapefiles_have_all_parts from zipfile import is_zipfile, ZipFile import tempfile import logging @@ -11,12 +12,6 @@ logger = logging.getLogger(__name__) -class XXXUploadFileForm(forms.ModelForm): - - class Meta: - model = UploadFile - fields = ['file'] - class UploadFileForm(forms.Form): file = forms.FileField(widget=forms.ClearableFileInput(attrs={'multiple': True})) @@ -24,42 +19,6 @@ class Meta: model = UploadFile fields = ['file'] - def validate_extension(self,filename): - base, extension = os.path.splitext(filename) - extension = extension.lstrip('.').lower() - if extension not in settings.OSGEO_IMPORTER_VALID_EXTENSIONS: - logger.debug('Validation Error: %s files not accepted by importer. file: %s',extension,filename) - self.add_error('file',"%s files not accepted by importer. file: %s" % (extension, filename)) - - def validate_shapefile_has_all_parts(self,validfiles): - shp = [] - prj = [] - dbf = [] - shx = [] - for file in validfiles: - base, extension = os.path.splitext(file) - extension = extension.lstrip('.').lower() - if extension == 'shp': - shp.append(base) - elif extension == 'prj': - prj.append(base) - elif extension == 'dbf': - dbf.append(base) - elif extension == 'shx': - shx.append(base) - if set(shp) == set(prj) == set(dbf) == set(shx): - return True - else: - logger.debug('Validation Error: All Shapefiles must include .shp, .prj, .dbf, and .shx') - self.add_error('file',"All Shapefiles must include .shp, .prj, .dbf, and .shx") - - def validate_inspector_can_read(self,filename): - try: - importer = load_handler(OSGEO_IMPORTER, filename) - data, inspector = importer.open_source_datastore(filename) - except NoDataSourceFound: - self.add_error('file','Unable to open file: %s' % filename) - def clean(self): logger.debug('..Cleaning...') cleaned_data = super(UploadFileForm, self).clean() @@ -70,19 +29,25 @@ def clean(self): # Create list of all potentially valid files, exploding first level zip files for file in files: logger.debug('cleaning %s',file.name) - self.validate_extension(file.name) + if not validate_extension(file.name): + self.add_error('file','Filetype not supported.') + logger.debug('Filetype not supported') + continue if is_zipfile(file): with ZipFile(file) as zip: for zipname in zip.namelist(): logger.debug('Checking extensions in zipfile %s file %s',file,zipname) - self.validate_extension(zipname) + if not validate_extension(zipname): + self.add_error('file','Filetype in zip not supported.') + continue validfiles.append(zipname) else: validfiles.append(file.name) # Make sure shapefiles have all their parts - self.validate_shapefile_has_all_parts(validfiles) + if not validate_shapefiles_have_all_parts(validfiles): + self.add_error('file','Shapefiles must include .shp,.dbf,.shx,.prj') logger.debug('valid files found: %s',validfiles) # Unpack all zip files and create list of cleaned file objects cleaned_files = [] @@ -103,10 +68,14 @@ def clean(self): cleaned_files.append(outfile) # After moving files in place make sure they can be opened by inspector + inspected_files = [] for cleaned_file in cleaned_files: - logger.debug('About to inspect %s in form',os.path.join(outputdir,file.name)) - self.validate_inspector_can_read(os.path.join(outputdir,file.name)) + logger.debug('About to inspect %s in form',os.path.join(outputdir,cleaned_file.name)) + if not validate_inspector_can_read(os.path.join(outputdir,file.name)): + self.add_error('file','Inspector could not read file or file is empty') + continue + inspected_files.append(cleaned_file) - logger.debug('cleaned_files %s',cleaned_files) - cleaned_data['file'] = cleaned_files + logger.debug('inspected_files %s',inspected_files) + cleaned_data['file'] = inspected_files return cleaned_data diff --git a/osgeo_importer/handlers/__init__.py b/osgeo_importer/handlers/__init__.py index 35f3c78..81651a5 100644 --- a/osgeo_importer/handlers/__init__.py +++ b/osgeo_importer/handlers/__init__.py @@ -11,7 +11,8 @@ 'osgeo_importer.handlers.geoserver.GeoServerBoundsHandler', 'osgeo_importer.handlers.geoserver.GenericSLDHandler', 'osgeo_importer.handlers.geonode.GeoNodePublishHandler', - 'osgeo_importer.handlers.geoserver.GeoServerStyleHandler'] + 'osgeo_importer.handlers.geoserver.GeoServerStyleHandler', + 'osgeo_importer.handlers.geonode.GeoNodeMetadataHandler'] IMPORT_HANDLERS = getattr(settings, 'IMPORT_HANDLERS', DEFAULT_IMPORT_HANDLERS) diff --git a/osgeo_importer/handlers/geonode/__init__.py b/osgeo_importer/handlers/geonode/__init__.py index c91da69..3c5219c 100644 --- a/osgeo_importer/handlers/geonode/__init__.py +++ b/osgeo_importer/handlers/geonode/__init__.py @@ -2,6 +2,7 @@ from osgeo_importer.models import UploadLayer from osgeo_importer.handlers import ImportHandlerMixin from osgeo_importer.handlers import ensure_can_run +from osgeo_importer.importers import UPLOAD_DIR from geonode.geoserver.helpers import gs_slurp from django.contrib.auth import get_user_model from django.conf import settings @@ -70,3 +71,55 @@ def handle(self, layer, layer_config, *args, **kwargs): upload_layer.save() return results + +class GeoNodeMetadataHandler(ImportHandlerMixin): + """ + Import uploaded XML + """ + + def can_run(self, layer, layer_config, *args, **kwargs): + """ + Only run this handler if the layer is found in Geoserver and the layer's style is the generic style. + """ + if not layer_config.get('metadata', None): + # logger.debug('Could not find any metadata xml in config %s', layer_config) + return False + + return True + + @ensure_can_run + def handle(self, layer, layer_config, *args, **kwargs): + """ + Update metadata from xml + """ + geonode_layer = Layer.objects.get(name=layer) + path = os.path.join(UPLOAD_DIR, str(self.importer.upload_file.upload.id)) + xmlfile = os.path.join(path, layer_config.get('metadata')) + geonode_layer.metadata_uploaded = True + identifier, vals, regions, keywords = set_metadata(open(xmlfile).read()) + + regions_resolved, regions_unresolved = resolve_regions(regions) + keywords.extend(regions_unresolved) + + # set regions + regions_resolved = list(set(regions_resolved)) + if regions: + if len(regions) > 0: + geonode_layer.regions.add(*regions_resolved) + + # set taggit keywords + keywords = list(set(keywords)) + geonode_layer.keywords.add(*keywords) + + # set model properties + for (key, value) in vals.items(): + if key == "spatial_representation_type": + # value = SpatialRepresentationType.objects.get(identifier=value) + pass + else: + logger.debug('Adding Metadata %s %s %s', geonode_layer, key, value) + setattr(geonode_layer, key, value) + + geonode_layer.save() + + return geonode_layer \ No newline at end of file diff --git a/osgeo_importer/handlers/geoserver/__init__.py b/osgeo_importer/handlers/geoserver/__init__.py index 8beb9a4..e8278f7 100644 --- a/osgeo_importer/handlers/geoserver/__init__.py +++ b/osgeo_importer/handlers/geoserver/__init__.py @@ -5,6 +5,7 @@ from django import db from django.conf import settings from osgeo_importer.handlers import ImportHandlerMixin, GetModifiedFieldsMixin, ensure_can_run +from osgeo_importer.importers import UPLOAD_DIR from geoserver.catalog import FailedRequestError, ConflictingDataError from geonode.geoserver.helpers import gs_catalog from geoserver.support import DimensionInfo @@ -361,7 +362,7 @@ def handle(self, layer, layer_config, *args, **kwargs): "slds": SLDS to add to layer """ lyr = self.catalog.get_layer(layer) - path = os.path.join(FileSystemStorage().location,'osgeo_importer_uploads', str(self.importer.upload_file.upload.id)) + path = os.path.join(UPLOAD_DIR, str(self.importer.upload_file.upload.id)) default_sld = layer_config.get('default_style', None) slds = layer_config.get('styles', None) all_slds = [] diff --git a/osgeo_importer/importers.py b/osgeo_importer/importers.py index 171282d..b438826 100644 --- a/osgeo_importer/importers.py +++ b/osgeo_importer/importers.py @@ -17,14 +17,25 @@ from django.conf import settings from django import db import logging +from django.core.files.storage import FileSystemStorage + logger = logging.getLogger(__name__) ogr.UseExceptions() - +MEDIA_ROOT = getattr(settings, 'MEDIA_ROOT', FileSystemStorage().location) OSGEO_IMPORTER = getattr(settings, 'OSGEO_IMPORTER', 'osgeo_importer.importers.OGRImport') -RASTER_FILES = getattr(settings, 'RASTER_FILES', '/tmp') +VALID_EXTENSIONS = getattr(settings, 'OSGEO_IMPORTER_VALID_EXTENSIONS', + ['shp','shx','prj','dbf','kml','geojson','json', + 'tif','tiff','gpkg','csv','zip','xml','sld']) +RASTER_FILES = getattr(settings, 'OSGEO_IMPORTER_RASTER_FILES', os.path.join(MEDIA_ROOT,'osgeo_importer_raster')) +UPLOAD_DIR = getattr(settings, 'OSGEO_IMPORTER_UPLOAD_DIR', os.path.join(MEDIA_ROOT,'osgeo_importer_uploads')) + +if not os.path.exists(RASTER_FILES): + os.makedirs(RASTER_FILES) +if not os.path.exists(UPLOAD_DIR): + os.makedirs(UPLOAD_DIR) class Import(object): """ @@ -37,8 +48,7 @@ class Import(object): enabled_handlers = IMPORT_HANDLERS source_inspectors = [] target_inspectors = [] - valid_extensions = ['gpx', 'geojson', 'json', 'zip', 'tar', 'kml', 'csv', 'shp', - 'tif', 'tiff', 'geotiff', 'gpkg'] + valid_extensions = VALID_EXTENSIONS def filter_handler_results(self, handler_name): """ diff --git a/osgeo_importer/validators.py b/osgeo_importer/validators.py new file mode 100644 index 0000000..421e109 --- /dev/null +++ b/osgeo_importer/validators.py @@ -0,0 +1,62 @@ +import os +from .utils import NoDataSourceFound, load_handler +from .importers import OSGEO_IMPORTER +import logging +from django.conf import settings + +logger = logging.getLogger(__name__) + +VALID_EXTENSIONS = getattr(settings, 'OSGEO_IMPORTER_VALID_EXTENSIONS', + ['shp','shx','prj','dbf','kml','geojson','json', + 'tif','tiff','gpkg','csv','zip','xml','sld']) + +NONDATA_EXTENSIONS = ['shx','prj','dbf','xml','sld'] + +def validate_extension(filename): + logger.debug('Checking Filename %s For Valid Extension',filename) + filedir, file = os.path.split(filename) + base, extension = os.path.splitext(file) + extension = extension.lstrip('.').lower() + logger.debug('Extension: %s %s', extension, VALID_EXTENSIONS) + if extension not in VALID_EXTENSIONS: + return False + else: + return True + +def validate_shapefiles_have_all_parts(filenamelist): + shp = [] + prj = [] + dbf = [] + shx = [] + for file in filenamelist: + base, extension = os.path.splitext(file) + extension = extension.lstrip('.').lower() + if extension == 'shp': + shp.append(base) + elif extension == 'prj': + prj.append(base) + elif extension == 'dbf': + dbf.append(base) + elif extension == 'shx': + shx.append(base) + if set(shp) == set(prj) == set(dbf) == set(shx): + return True + else: + return False + +def validate_inspector_can_read(filename): + filedir, file = os.path.split(filename) + base, extension = os.path.splitext(file) + extension = extension.lstrip('.').lower() + if extension in NONDATA_EXTENSIONS: + return True + try: + importer = load_handler(OSGEO_IMPORTER, filename) + data, inspector = importer.open_source_datastore(filename) + # Ensure the data has a geometry. + for description in inspector.describe_fields(): + if description.get('raster') == False and description.get('geom_type') in inspector.INVALID_GEOMETRY_TYPES: + return False + except NoDataSourceFound: + return False + return True From 22cc4d1af451c8c7343f3bdc807f8da2011c7b1d Mon Sep 17 00:00:00 2001 From: "david.w.bitner" Date: Mon, 29 Aug 2016 16:20:33 -0500 Subject: [PATCH 04/27] all tests passing with new multi-uploader --- osgeo_importer/api.py | 2 +- osgeo_importer/forms.py | 7 --- osgeo_importer/handlers/geonode/__init__.py | 6 +-- osgeo_importer/handlers/geoserver/__init__.py | 1 + osgeo_importer/models.py | 44 +++++++++---------- osgeo_importer/tests.py | 12 +++-- osgeo_importer/validators.py | 2 - osgeo_importer/views.py | 19 +++----- 8 files changed, 41 insertions(+), 52 deletions(-) diff --git a/osgeo_importer/api.py b/osgeo_importer/api.py index 9005461..5ee98d3 100644 --- a/osgeo_importer/api.py +++ b/osgeo_importer/api.py @@ -80,7 +80,7 @@ def import_layer(self, request, **kwargs): if not configuration_options: raise ImmediateHttpResponse(response=http.HttpBadRequest('Configuration options missing.')) - uploaded_file = obj.upload.uploadfile_set.first() + uploaded_file = obj.upload_file import_result = import_object.delay(uploaded_file.id, configuration_options=configuration_options) # query the db again for this object since it may have been updated during the import diff --git a/osgeo_importer/forms.py b/osgeo_importer/forms.py index b95740b..9665ad3 100644 --- a/osgeo_importer/forms.py +++ b/osgeo_importer/forms.py @@ -28,16 +28,13 @@ def clean(self): # Create list of all potentially valid files, exploding first level zip files for file in files: - logger.debug('cleaning %s',file.name) if not validate_extension(file.name): self.add_error('file','Filetype not supported.') - logger.debug('Filetype not supported') continue if is_zipfile(file): with ZipFile(file) as zip: for zipname in zip.namelist(): - logger.debug('Checking extensions in zipfile %s file %s',file,zipname) if not validate_extension(zipname): self.add_error('file','Filetype in zip not supported.') continue @@ -48,11 +45,9 @@ def clean(self): # Make sure shapefiles have all their parts if not validate_shapefiles_have_all_parts(validfiles): self.add_error('file','Shapefiles must include .shp,.dbf,.shx,.prj') - logger.debug('valid files found: %s',validfiles) # Unpack all zip files and create list of cleaned file objects cleaned_files = [] for file in files: - logger.debug('file: %s',file) if file.name in validfiles: with open(os.path.join(outputdir,file.name),'w') as outfile: for chunk in file.chunks(): @@ -70,12 +65,10 @@ def clean(self): # After moving files in place make sure they can be opened by inspector inspected_files = [] for cleaned_file in cleaned_files: - logger.debug('About to inspect %s in form',os.path.join(outputdir,cleaned_file.name)) if not validate_inspector_can_read(os.path.join(outputdir,file.name)): self.add_error('file','Inspector could not read file or file is empty') continue inspected_files.append(cleaned_file) - logger.debug('inspected_files %s',inspected_files) cleaned_data['file'] = inspected_files return cleaned_data diff --git a/osgeo_importer/handlers/geonode/__init__.py b/osgeo_importer/handlers/geonode/__init__.py index 3c5219c..b89b88b 100644 --- a/osgeo_importer/handlers/geonode/__init__.py +++ b/osgeo_importer/handlers/geonode/__init__.py @@ -4,6 +4,8 @@ from osgeo_importer.handlers import ensure_can_run from osgeo_importer.importers import UPLOAD_DIR from geonode.geoserver.helpers import gs_slurp +from geonode.layers.metadata import set_metadata +from geonode.layers.utils import resolve_regions from django.contrib.auth import get_user_model from django.conf import settings from django import db @@ -65,7 +67,7 @@ def handle(self, layer, layer_config, *args, **kwargs): if self.importer.upload_file and results['layers'][0]['status'] == 'created': matched_layer = Layer.objects.get(name=results['layers'][0]['name']) - upload_layer = UploadLayer.objects.get(upload=self.importer.upload_file.upload, + upload_layer = UploadLayer.objects.get(upload_file=self.importer.upload_file.pk, index=layer_config.get('index')) upload_layer.layer = matched_layer upload_layer.save() @@ -82,7 +84,6 @@ def can_run(self, layer, layer_config, *args, **kwargs): Only run this handler if the layer is found in Geoserver and the layer's style is the generic style. """ if not layer_config.get('metadata', None): - # logger.debug('Could not find any metadata xml in config %s', layer_config) return False return True @@ -117,7 +118,6 @@ def handle(self, layer, layer_config, *args, **kwargs): # value = SpatialRepresentationType.objects.get(identifier=value) pass else: - logger.debug('Adding Metadata %s %s %s', geonode_layer, key, value) setattr(geonode_layer, key, value) geonode_layer.save() diff --git a/osgeo_importer/handlers/geoserver/__init__.py b/osgeo_importer/handlers/geoserver/__init__.py index e8278f7..81193a2 100644 --- a/osgeo_importer/handlers/geoserver/__init__.py +++ b/osgeo_importer/handlers/geoserver/__init__.py @@ -8,6 +8,7 @@ from osgeo_importer.importers import UPLOAD_DIR from geoserver.catalog import FailedRequestError, ConflictingDataError from geonode.geoserver.helpers import gs_catalog +from geonode.layers.utils import resolve_regions from geoserver.support import DimensionInfo from django.core.files.storage import FileSystemStorage from osgeo_importer.utils import increment_filename diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index 0359c38..eabb2eb 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -31,7 +31,6 @@ from djcelery.models import TaskState from jsonfield import JSONField - try: from django.contrib.contenttypes.fields import GenericForeignKey except ImportError: @@ -179,11 +178,33 @@ def __unicode__(self): return 'Upload [%s] %s, %s' % (self.id, self.name, self.user) +class UploadFile(models.Model): + upload = models.ForeignKey(UploadedData, null=True, blank=True) + file = models.FileField(upload_to="uploads", validators=[validate_file_extension, validate_inspector_can_read]) + slug = models.SlugField(max_length=250, blank=True) + + def __unicode__(self): + return self.slug + + @property + def name(self): + return os.path.basename(self.file.path) + + def save(self, *args, **kwargs): + self.slug = self.file.name + super(UploadFile, self).save(*args, **kwargs) + + def delete(self, *args, **kwargs): + self.file.delete(False) + super(UploadFile, self).delete(*args, **kwargs) + + class UploadLayer(models.Model): """ Layers stored in an uploaded data set. """ upload = models.ForeignKey(UploadedData, null=True, blank=True) + upload_file = models.ForeignKey(UploadFile, null=True, blank=True) index = models.IntegerField(default=0) name = models.CharField(max_length=64, null=True) fields = JSONField(null=True, default={}) @@ -237,27 +258,6 @@ class Meta: ordering = ('index',) -class UploadFile(models.Model): - upload = models.ForeignKey(UploadedData, null=True, blank=True) - file = models.FileField(upload_to="uploads", validators=[validate_file_extension, validate_inspector_can_read]) - slug = models.SlugField(max_length=250, blank=True) - - def __unicode__(self): - return self.slug - - @property - def name(self): - return os.path.basename(self.file.path) - - def save(self, *args, **kwargs): - self.slug = self.file.name - super(UploadFile, self).save(*args, **kwargs) - - def delete(self, *args, **kwargs): - self.file.delete(False) - super(UploadFile, self).delete(*args, **kwargs) - - class UploadException(models.Model): """ A generic object for storing exceptions during upload diff --git a/osgeo_importer/tests.py b/osgeo_importer/tests.py index 490fac5..631be00 100644 --- a/osgeo_importer/tests.py +++ b/osgeo_importer/tests.py @@ -184,8 +184,6 @@ def generic_api_upload(self, files, configuration_options=None): upload_layers = UploadLayer.objects.filter(upload_id=uploadid) for upload_layer in upload_layers: - print upload_layer.id - print upload_layer.name for config in configuration_options: if config['upload_file_name'] == upload_layer.name: payload = config['config'] @@ -269,8 +267,14 @@ def test_upload_with_metadata(self): ] ) self.assertEqual(5, upload['count']) - print upload - layerid = upload['uploaded'][0]['pk']; + uploadid = upload['id'] + uploadobj = UploadedData.objects.get(pk=uploadid) + uplayers = UploadLayer.objects.filter(upload=uploadid) + layerid = uplayers[0].pk + + upfiles_cnt = UploadFile.objects.filter(upload=uploadid).count() + self.assertEqual(5,upfiles_cnt) + layer = Layer.objects.get(pk=layerid) self.assertEqual(layer.language, 'eng') self.assertEqual(layer.title, 'Old_Americas_LSIB_Polygons_Detailed_2013Mar') diff --git a/osgeo_importer/validators.py b/osgeo_importer/validators.py index 421e109..9b3526f 100644 --- a/osgeo_importer/validators.py +++ b/osgeo_importer/validators.py @@ -13,11 +13,9 @@ NONDATA_EXTENSIONS = ['shx','prj','dbf','xml','sld'] def validate_extension(filename): - logger.debug('Checking Filename %s For Valid Extension',filename) filedir, file = os.path.split(filename) base, extension = os.path.splitext(file) extension = extension.lstrip('.').lower() - logger.debug('Extension: %s %s', extension, VALID_EXTENSIONS) if extension not in VALID_EXTENSIONS: return False else: diff --git a/osgeo_importer/views.py b/osgeo_importer/views.py index 57d26f9..5006a7a 100644 --- a/osgeo_importer/views.py +++ b/osgeo_importer/views.py @@ -17,11 +17,6 @@ OSGEO_IMPORTER = import_string(OSGEO_IMPORTER) logger = logging.getLogger(__name__) -if logger.handlers: - for handler in logger.handlers: - logger.removeHandler(handler) -logging.basicConfig(filename='/vagrant/views.log', level=logging.DEBUG) - class JSONResponseMixin(object): @@ -94,15 +89,12 @@ def form_valid(self, form): outdir = os.path.join(FileSystemStorage().location,outpath) if not os.path.exists(outdir): os.makedirs(outdir) - logger.debug("%s",outdir) - logger.debug("cleaned_data: %s",form.cleaned_data) # Move all files to uploads directory using upload pk # Must be done for all files before saving upfile for validation finalfiles = [] for each in form.cleaned_data['file']: tofile = os.path.join(outdir,os.path.basename(each.name)) - logger.debug('moving %s to %s', each.name, tofile) shutil.move(each.name, tofile) finalfiles.append(tofile) @@ -119,11 +111,12 @@ def form_valid(self, form): for layer in description: configuration_options = DEFAULT_LAYER_CONFIGURATION.copy() configuration_options.update({'index': layer.get('index')}) - upload.uploadlayer_set.add(UploadLayer(name=upfile_basename, - fields=layer.get('fields', {}), - index=layer.get('index'), - feature_count=layer.get('feature_count',None), - configuration_options=configuration_options)) + upload.uploadlayer_set.add(UploadLayer(upload_file=upfile, + name=upfile_basename, + fields=layer.get('fields', {}), + index=layer.get('index'), + feature_count=layer.get('feature_count',None), + configuration_options=configuration_options)) upload.complete = True upload.state = 'UPLOADED' upload.save() From f3d512c206a762a8426fa65793a5ab488e849b40 Mon Sep 17 00:00:00 2001 From: "david.w.bitner" Date: Mon, 29 Aug 2016 16:48:40 -0500 Subject: [PATCH 05/27] flake cleanup --- osgeo_importer/forms.py | 20 +++++++++---------- osgeo_importer/handlers/geonode/__init__.py | 3 ++- osgeo_importer/handlers/geoserver/__init__.py | 2 -- osgeo_importer/importers.py | 11 +++++----- osgeo_importer/models.py | 1 - osgeo_importer/validators.py | 12 +++++------ osgeo_importer/views.py | 17 ++++++++-------- 7 files changed, 30 insertions(+), 36 deletions(-) diff --git a/osgeo_importer/forms.py b/osgeo_importer/forms.py index 9665ad3..25bee13 100644 --- a/osgeo_importer/forms.py +++ b/osgeo_importer/forms.py @@ -1,17 +1,15 @@ import os from django import forms from .models import UploadFile -from .utils import NoDataSourceFound, load_handler -from .importers import OSGEO_IMPORTER from .validators import validate_extension, validate_inspector_can_read, validate_shapefiles_have_all_parts from zipfile import is_zipfile, ZipFile import tempfile import logging import shutil -from django.conf import settings logger = logging.getLogger(__name__) + class UploadFileForm(forms.Form): file = forms.FileField(widget=forms.ClearableFileInput(attrs={'multiple': True})) @@ -29,14 +27,14 @@ def clean(self): # Create list of all potentially valid files, exploding first level zip files for file in files: if not validate_extension(file.name): - self.add_error('file','Filetype not supported.') + self.add_error('file', 'Filetype not supported.') continue if is_zipfile(file): with ZipFile(file) as zip: for zipname in zip.namelist(): if not validate_extension(zipname): - self.add_error('file','Filetype in zip not supported.') + self.add_error('file', 'Filetype in zip not supported.') continue validfiles.append(zipname) else: @@ -44,12 +42,12 @@ def clean(self): # Make sure shapefiles have all their parts if not validate_shapefiles_have_all_parts(validfiles): - self.add_error('file','Shapefiles must include .shp,.dbf,.shx,.prj') + self.add_error('file', 'Shapefiles must include .shp,.dbf,.shx,.prj') # Unpack all zip files and create list of cleaned file objects cleaned_files = [] for file in files: if file.name in validfiles: - with open(os.path.join(outputdir,file.name),'w') as outfile: + with open(os.path.join(outputdir, file.name), 'w') as outfile: for chunk in file.chunks(): outfile.write(chunk) cleaned_files.append(outfile) @@ -58,15 +56,15 @@ def clean(self): for zipfile in zip.namelist(): if zipfile in validfiles: with zip.open(zipfile) as f: - with open(os.path.join(outputdir,zipfile),'w') as outfile: - shutil.copyfileobj(f,outfile) + with open(os.path.join(outputdir, zipfile), 'w') as outfile: + shutil.copyfileobj(f, outfile) cleaned_files.append(outfile) # After moving files in place make sure they can be opened by inspector inspected_files = [] for cleaned_file in cleaned_files: - if not validate_inspector_can_read(os.path.join(outputdir,file.name)): - self.add_error('file','Inspector could not read file or file is empty') + if not validate_inspector_can_read(os.path.join(outputdir, file.name)): + self.add_error('file', 'Inspector could not read file or file is empty') continue inspected_files.append(cleaned_file) diff --git a/osgeo_importer/handlers/geonode/__init__.py b/osgeo_importer/handlers/geonode/__init__.py index b89b88b..1adaeb3 100644 --- a/osgeo_importer/handlers/geonode/__init__.py +++ b/osgeo_importer/handlers/geonode/__init__.py @@ -74,6 +74,7 @@ def handle(self, layer, layer_config, *args, **kwargs): return results + class GeoNodeMetadataHandler(ImportHandlerMixin): """ Import uploaded XML @@ -122,4 +123,4 @@ def handle(self, layer, layer_config, *args, **kwargs): geonode_layer.save() - return geonode_layer \ No newline at end of file + return geonode_layer diff --git a/osgeo_importer/handlers/geoserver/__init__.py b/osgeo_importer/handlers/geoserver/__init__.py index 81193a2..7d4902a 100644 --- a/osgeo_importer/handlers/geoserver/__init__.py +++ b/osgeo_importer/handlers/geoserver/__init__.py @@ -8,9 +8,7 @@ from osgeo_importer.importers import UPLOAD_DIR from geoserver.catalog import FailedRequestError, ConflictingDataError from geonode.geoserver.helpers import gs_catalog -from geonode.layers.utils import resolve_regions from geoserver.support import DimensionInfo -from django.core.files.storage import FileSystemStorage from osgeo_importer.utils import increment_filename diff --git a/osgeo_importer/importers.py b/osgeo_importer/importers.py index b438826..0449cab 100644 --- a/osgeo_importer/importers.py +++ b/osgeo_importer/importers.py @@ -25,11 +25,11 @@ MEDIA_ROOT = getattr(settings, 'MEDIA_ROOT', FileSystemStorage().location) OSGEO_IMPORTER = getattr(settings, 'OSGEO_IMPORTER', 'osgeo_importer.importers.OGRImport') -VALID_EXTENSIONS = getattr(settings, 'OSGEO_IMPORTER_VALID_EXTENSIONS', - ['shp','shx','prj','dbf','kml','geojson','json', - 'tif','tiff','gpkg','csv','zip','xml','sld']) -RASTER_FILES = getattr(settings, 'OSGEO_IMPORTER_RASTER_FILES', os.path.join(MEDIA_ROOT,'osgeo_importer_raster')) -UPLOAD_DIR = getattr(settings, 'OSGEO_IMPORTER_UPLOAD_DIR', os.path.join(MEDIA_ROOT,'osgeo_importer_uploads')) +VALID_EXTENSIONS = getattr(settings, 'OSGEO_IMPORTER_VALID_EXTENSIONS', + ['shp', 'shx', 'prj', 'dbf', 'kml', 'geojson', 'json', + 'tif', 'tiff', 'gpkg', 'csv', 'zip', 'xml', 'sld']) +RASTER_FILES = getattr(settings, 'OSGEO_IMPORTER_RASTER_FILES', os.path.join(MEDIA_ROOT, 'osgeo_importer_raster')) +UPLOAD_DIR = getattr(settings, 'OSGEO_IMPORTER_UPLOAD_DIR', os.path.join(MEDIA_ROOT, 'osgeo_importer_uploads')) if not os.path.exists(RASTER_FILES): os.makedirs(RASTER_FILES) @@ -37,6 +37,7 @@ if not os.path.exists(UPLOAD_DIR): os.makedirs(UPLOAD_DIR) + class Import(object): """ Importers are responsible for opening incoming geospatial datasets (using one or many inspectors) and diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index eabb2eb..e6c6c54 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -78,7 +78,6 @@ def validate_inspector_can_read(value): """ Validates Geospatial data. """ - logger.debug('Inspecting Can Read %s',value.name) name = value.name root, extension = os.path.splitext(name.lower()) diff --git a/osgeo_importer/validators.py b/osgeo_importer/validators.py index 9b3526f..1aca078 100644 --- a/osgeo_importer/validators.py +++ b/osgeo_importer/validators.py @@ -1,16 +1,12 @@ import os from .utils import NoDataSourceFound, load_handler -from .importers import OSGEO_IMPORTER +from .importers import OSGEO_IMPORTER, VALID_EXTENSIONS import logging -from django.conf import settings logger = logging.getLogger(__name__) -VALID_EXTENSIONS = getattr(settings, 'OSGEO_IMPORTER_VALID_EXTENSIONS', - ['shp','shx','prj','dbf','kml','geojson','json', - 'tif','tiff','gpkg','csv','zip','xml','sld']) +NONDATA_EXTENSIONS = ['shx', 'prj', 'dbf', 'xml', 'sld'] -NONDATA_EXTENSIONS = ['shx','prj','dbf','xml','sld'] def validate_extension(filename): filedir, file = os.path.split(filename) @@ -21,6 +17,7 @@ def validate_extension(filename): else: return True + def validate_shapefiles_have_all_parts(filenamelist): shp = [] prj = [] @@ -42,6 +39,7 @@ def validate_shapefiles_have_all_parts(filenamelist): else: return False + def validate_inspector_can_read(filename): filedir, file = os.path.split(filename) base, extension = os.path.splitext(file) @@ -53,7 +51,7 @@ def validate_inspector_can_read(filename): data, inspector = importer.open_source_datastore(filename) # Ensure the data has a geometry. for description in inspector.describe_fields(): - if description.get('raster') == False and description.get('geom_type') in inspector.INVALID_GEOMETRY_TYPES: + if description.get('raster') is False and description.get('geom_type') in inspector.INVALID_GEOMETRY_TYPES: return False except NoDataSourceFound: return False diff --git a/osgeo_importer/views.py b/osgeo_importer/views.py index 5006a7a..f0a4f7b 100644 --- a/osgeo_importer/views.py +++ b/osgeo_importer/views.py @@ -10,7 +10,6 @@ from .importers import OSGEO_IMPORTER from .inspectors import OSGEO_INSPECTOR from .utils import import_string -from django.conf import settings from django.core.files.storage import FileSystemStorage OSGEO_INSPECTOR = import_string(OSGEO_INSPECTOR) @@ -85,28 +84,27 @@ def form_valid(self, form): upload.save() # Create Upload Directory based on Upload PK - outpath = os.path.join('osgeo_importer_uploads',str(upload.pk)) - outdir = os.path.join(FileSystemStorage().location,outpath) + outpath = os.path.join('osgeo_importer_uploads', str(upload.pk)) + outdir = os.path.join(FileSystemStorage().location, outpath) if not os.path.exists(outdir): os.makedirs(outdir) - + # Move all files to uploads directory using upload pk # Must be done for all files before saving upfile for validation finalfiles = [] for each in form.cleaned_data['file']: - tofile = os.path.join(outdir,os.path.basename(each.name)) + tofile = os.path.join(outdir, os.path.basename(each.name)) shutil.move(each.name, tofile) finalfiles.append(tofile) # Loop through and create uploadfiles and uploadlayers - uplayers=[] for each in finalfiles: upfile = UploadFile.objects.create(upload=upload) upfile.file.name = each upfile.save() upfile_basename = os.path.basename(each) upfile_root, upfile_ext = os.path.splitext(upfile_basename) - if upfile_ext.lower() not in ['.prj','.dbf','.shx']: + if upfile_ext.lower() not in ['.prj', '.dbf', '.shx']: description = self.get_fields(each) for layer in description: configuration_options = DEFAULT_LAYER_CONFIGURATION.copy() @@ -115,14 +113,15 @@ def form_valid(self, form): name=upfile_basename, fields=layer.get('fields', {}), index=layer.get('index'), - feature_count=layer.get('feature_count',None), + feature_count=layer.get('feature_count', None), configuration_options=configuration_options)) upload.complete = True upload.state = 'UPLOADED' upload.save() if self.json: - return self.render_to_json_response({'state': upload.state, 'id': upload.id, 'count': UploadFile.objects.filter(upload=upload.id).count()}) + return self.render_to_json_response({'state': upload.state, 'id': upload.id, + 'count': UploadFile.objects.filter(upload=upload.id).count()}) return super(FileAddView, self).form_valid(form) From 6df68116615d97a9cd452b7589c1ba87c532a102 Mon Sep 17 00:00:00 2001 From: "david.w.bitner" Date: Tue, 30 Aug 2016 10:52:40 -0500 Subject: [PATCH 06/27] add migration --- .../0003_uploadlayer_upload_file.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 osgeo_importer/migrations/0003_uploadlayer_upload_file.py diff --git a/osgeo_importer/migrations/0003_uploadlayer_upload_file.py b/osgeo_importer/migrations/0003_uploadlayer_upload_file.py new file mode 100644 index 0000000..9028514 --- /dev/null +++ b/osgeo_importer/migrations/0003_uploadlayer_upload_file.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('osgeo_importer', '0002_auto_20160713_1429'), + ] + + operations = [ + migrations.AddField( + model_name='uploadlayer', + name='upload_file', + field=models.ForeignKey(blank=True, to='osgeo_importer.UploadFile', null=True), + ), + ] From b6b790eccf55ae55e1d5fff7c3c490e1252cbbcb Mon Sep 17 00:00:00 2001 From: "david.w.bitner" Date: Tue, 30 Aug 2016 14:49:31 -0500 Subject: [PATCH 07/27] calculate total size for upload, rather than first file --- osgeo_importer/models.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index e6c6c54..c023528 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -150,22 +150,20 @@ def filesize(self): """ Humanizes the upload file size. """ - if not self.size: - uploaded_file = self.uploadfile_set.first() - - if uploaded_file: - self.size = uploaded_file.file.size - self.save() - else: - return - + size = 0 + if not self.size or self.size == 0: + if self.uploadfile_set.count() > 0: + for uf in self.uploadfile_set.all(): + size += uf.file.size + self.size = size + self.save() return sizeof_fmt(self.size) def file_url(self): """ Exposes the file url. """ - return self.uploadfile_set.first().file.url + return '' # self.uploadfile_set.first().file.url def any_layers_imported(self): return any(self.uploadlayer_set.all().values_list('layer', flat=True)) From 0c99c737f80479b04c5b0122d3a694d6098d935a Mon Sep 17 00:00:00 2001 From: "david.w.bitner" Date: Tue, 13 Sep 2016 16:09:00 -0500 Subject: [PATCH 08/27] flake 8 fix --- osgeo_importer/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index c023528..3ea66d5 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -163,7 +163,7 @@ def file_url(self): """ Exposes the file url. """ - return '' # self.uploadfile_set.first().file.url + return '' # self.uploadfile_set.first().file.url def any_layers_imported(self): return any(self.uploadlayer_set.all().values_list('layer', flat=True)) From 3bf7f584118b0c5fbe7c352cd80ce84edb52148f Mon Sep 17 00:00:00 2001 From: "david.w.bitner" Date: Fri, 16 Sep 2016 13:38:22 -0500 Subject: [PATCH 09/27] fix cleaned_file reference in form validation --- osgeo_importer/forms.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/osgeo_importer/forms.py b/osgeo_importer/forms.py index 25bee13..25822e2 100644 --- a/osgeo_importer/forms.py +++ b/osgeo_importer/forms.py @@ -6,7 +6,6 @@ import tempfile import logging import shutil - logger = logging.getLogger(__name__) @@ -18,7 +17,6 @@ class Meta: fields = ['file'] def clean(self): - logger.debug('..Cleaning...') cleaned_data = super(UploadFileForm, self).clean() outputdir = tempfile.mkdtemp() files = self.files.getlist('file') @@ -39,7 +37,6 @@ def clean(self): validfiles.append(zipname) else: validfiles.append(file.name) - # Make sure shapefiles have all their parts if not validate_shapefiles_have_all_parts(validfiles): self.add_error('file', 'Shapefiles must include .shp,.dbf,.shx,.prj') @@ -63,8 +60,8 @@ def clean(self): # After moving files in place make sure they can be opened by inspector inspected_files = [] for cleaned_file in cleaned_files: - if not validate_inspector_can_read(os.path.join(outputdir, file.name)): - self.add_error('file', 'Inspector could not read file or file is empty') + if not validate_inspector_can_read(os.path.join(outputdir, cleaned_file.name)): + self.add_error('file', 'Inspector could not read file %s or file is empty'%(os.path.join(outputdir,file.name))) continue inspected_files.append(cleaned_file) From 01df9be1ceeec0588f5e402d3532e069c960bd0b Mon Sep 17 00:00:00 2001 From: Sasha Hart Date: Fri, 23 Sep 2016 19:44:30 -0500 Subject: [PATCH 10/27] re-enable tests that were commented out --- osgeo_importer/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osgeo_importer/tests.py b/osgeo_importer/tests.py index 631be00..2d03daf 100644 --- a/osgeo_importer/tests.py +++ b/osgeo_importer/tests.py @@ -624,10 +624,10 @@ def test_file_add_view(self): self.assertTrue(len(response.context['object_list']) == 1) upload = response.context['object_list'][0] self.assertEqual(upload.user.username, 'non_admin') - # self.assertEqual(upload.file_type, 'GeoJSON') + self.assertEqual(upload.file_type, 'GeoJSON') self.assertTrue(upload.uploadlayer_set.all()) self.assertEqual(upload.state, 'UPLOADED') - # self.assertIsNotNone(upload.name) + self.assertIsNotNone(upload.name) uploaded_file = upload.uploadfile_set.first() self.assertTrue(os.path.exists(uploaded_file.file.path)) From 66466cd870e1e35291131425c3ce857b08e36b13 Mon Sep 17 00:00:00 2001 From: Sara Safavi Date: Thu, 22 Sep 2016 18:38:11 -0500 Subject: [PATCH 11/27] Reformatted for flake8; also fixed "file" -> "cleaned_file" reference --- osgeo_importer/forms.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/osgeo_importer/forms.py b/osgeo_importer/forms.py index 25822e2..9908714 100644 --- a/osgeo_importer/forms.py +++ b/osgeo_importer/forms.py @@ -60,8 +60,12 @@ def clean(self): # After moving files in place make sure they can be opened by inspector inspected_files = [] for cleaned_file in cleaned_files: - if not validate_inspector_can_read(os.path.join(outputdir, cleaned_file.name)): - self.add_error('file', 'Inspector could not read file %s or file is empty'%(os.path.join(outputdir,file.name))) + cleaned_file_path = os.path.join(outputdir, cleaned_file.name) + if not validate_inspector_can_read(cleaned_file_path): + self.add_error( + 'file', + 'Inspector could not read file {} or file is empty'.format(cleaned_file_path) + ) continue inspected_files.append(cleaned_file) From 27e67717d3910261fc251618dfa77e6f1a9583b9 Mon Sep 17 00:00:00 2001 From: Sara Safavi Date: Thu, 22 Sep 2016 18:44:47 -0500 Subject: [PATCH 12/27] Minor template fix now that we allow multiple files in a single upload --- osgeo_importer/templates/osgeo_importer/new.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osgeo_importer/templates/osgeo_importer/new.html b/osgeo_importer/templates/osgeo_importer/new.html index a8884b4..b92b284 100644 --- a/osgeo_importer/templates/osgeo_importer/new.html +++ b/osgeo_importer/templates/osgeo_importer/new.html @@ -32,7 +32,7 @@

{% trans "Add data" %}

{{form}}
- +
From 66941047ef4f9353e8a2c181922a41aa9f42e8ed Mon Sep 17 00:00:00 2001 From: Sara Safavi Date: Thu, 22 Sep 2016 18:45:40 -0500 Subject: [PATCH 13/27] Remove outdated/partial "valid file types" verbiage from template --- osgeo_importer/templates/osgeo_importer/new.html | 3 --- 1 file changed, 3 deletions(-) diff --git a/osgeo_importer/templates/osgeo_importer/new.html b/osgeo_importer/templates/osgeo_importer/new.html index b92b284..bab7188 100644 --- a/osgeo_importer/templates/osgeo_importer/new.html +++ b/osgeo_importer/templates/osgeo_importer/new.html @@ -40,9 +40,6 @@

{% trans "Add data" %}

{{ error|escape }} {% endfor %}
-
- Valid file types are .csv, .geojson, .gpx, .kml, .tif, zipped Shapefile. -
From d386e0a78782ef493b1eaf72c63ff9ca7628a8ef Mon Sep 17 00:00:00 2001 From: Sasha Hart Date: Fri, 23 Sep 2016 14:26:00 -0500 Subject: [PATCH 14/27] various little refactors variable name: b -> bundle and other variable names: uploadid -> upload_id, uploadobj -> upload_obj, cnt -> count uf -> uploadfile make import_layer explicitly take pk kwarg: since this method is only using kwargs to get pk, we might as well extract pk early and default it to None instead of using kwargs.get. I actually would prefer pk to be a required arg and to dispense with **kwargs, so we have a real signature that can check us when we make mistakes; but that will have to depend on an analysis of where these methods are used pass request as Bundle arg instead of setting attr after --- osgeo_importer/api.py | 12 ++--- osgeo_importer/handlers/geonode/__init__.py | 6 +-- osgeo_importer/handlers/geoserver/__init__.py | 3 +- osgeo_importer/models.py | 15 +++--- osgeo_importer/tests.py | 51 +++++++++---------- osgeo_importer/views.py | 16 +++--- osgeo_importer_prj/settings.py | 5 +- 7 files changed, 53 insertions(+), 55 deletions(-) diff --git a/osgeo_importer/api.py b/osgeo_importer/api.py index 5ee98d3..986be50 100644 --- a/osgeo_importer/api.py +++ b/osgeo_importer/api.py @@ -50,17 +50,15 @@ def get_object_list(self, request): def clean_configuration_options(self, request, obj, configuration_options): return configuration_options - def import_layer(self, request, **kwargs): - """ - Imports a layer + def import_layer(self, request, pk=None, **kwargs): + """Imports a layer """ self.method_check(request, allowed=['post']) - b = Bundle() - b.request = request + bundle = Bundle(request=request) try: - obj = self.obj_get(b, pk=kwargs.get('pk')) + obj = self.obj_get(bundle, pk=pk) except UploadLayer.DoesNotExist: raise ImmediateHttpResponse(response=http.HttpNotFound()) @@ -84,7 +82,7 @@ def import_layer(self, request, **kwargs): import_result = import_object.delay(uploaded_file.id, configuration_options=configuration_options) # query the db again for this object since it may have been updated during the import - obj = self.obj_get(b, pk=kwargs.get('pk')) + obj = self.obj_get(bundle, pk=pk) obj.task_id = import_result.id obj.save() diff --git a/osgeo_importer/handlers/geonode/__init__.py b/osgeo_importer/handlers/geonode/__init__.py index 1adaeb3..ef00694 100644 --- a/osgeo_importer/handlers/geonode/__init__.py +++ b/osgeo_importer/handlers/geonode/__init__.py @@ -76,8 +76,7 @@ def handle(self, layer, layer_config, *args, **kwargs): class GeoNodeMetadataHandler(ImportHandlerMixin): - """ - Import uploaded XML + """Import uploaded XML """ def can_run(self, layer, layer_config, *args, **kwargs): @@ -91,8 +90,7 @@ def can_run(self, layer, layer_config, *args, **kwargs): @ensure_can_run def handle(self, layer, layer_config, *args, **kwargs): - """ - Update metadata from xml + """Update metadata from XML """ geonode_layer = Layer.objects.get(name=layer) path = os.path.join(UPLOAD_DIR, str(self.importer.upload_file.upload.id)) diff --git a/osgeo_importer/handlers/geoserver/__init__.py b/osgeo_importer/handlers/geoserver/__init__.py index 7d4902a..b00dbee 100644 --- a/osgeo_importer/handlers/geoserver/__init__.py +++ b/osgeo_importer/handlers/geoserver/__init__.py @@ -337,8 +337,7 @@ def handle(self, layer, layer_config, *args, **kwargs): class GeoServerStyleHandler(GeoserverHandlerMixin): - """ - Adds styles to GeoServer Layer + """Adds styles to GeoServer Layer """ catalog = gs_catalog catalog._cache.clear() diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index 3ea66d5..f3d620b 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -147,21 +147,19 @@ def get_delete_url(self): @property def filesize(self): - """ - Humanizes the upload file size. + """Humanizes the upload file size. """ size = 0 - if not self.size or self.size == 0: + if not self.size: if self.uploadfile_set.count() > 0: - for uf in self.uploadfile_set.all(): - size += uf.file.size + for uploadfile in self.uploadfile_set.all(): + size += uploadfile.file.size self.size = size self.save() return sizeof_fmt(self.size) def file_url(self): - """ - Exposes the file url. + """Exposes the file url. """ return '' # self.uploadfile_set.first().file.url @@ -197,8 +195,7 @@ def delete(self, *args, **kwargs): class UploadLayer(models.Model): - """ - Layers stored in an uploaded data set. + """Layers stored in an uploaded data set. """ upload = models.ForeignKey(UploadedData, null=True, blank=True) upload_file = models.ForeignKey(UploadFile, null=True, blank=True) diff --git a/osgeo_importer/tests.py b/osgeo_importer/tests.py index 2d03daf..6c199d5 100644 --- a/osgeo_importer/tests.py +++ b/osgeo_importer/tests.py @@ -148,12 +148,11 @@ def generic_import(self, file, configuration_options=[{'index': 0}]): return layer_results[0] def generic_api_upload(self, files, configuration_options=None): - """ - Tests the import api. + """Tests the import api. """ c = AdminClient() c.login_as_non_admin() - + # Upload Files if isinstance(files, type(str())): files = [files] @@ -180,18 +179,21 @@ def generic_api_upload(self, files, configuration_options=None): self.assertEqual(content['id'], 1) # Configure Uploaded Files - uploadid = content['id'] - upload_layers = UploadLayer.objects.filter(upload_id=uploadid) - + upload_id = content['id'] + upload_layers = UploadLayer.objects.filter(upload_id=upload_id) + for upload_layer in upload_layers: for config in configuration_options: if config['upload_file_name'] == upload_layer.name: payload = config['config'] - response = c.post('/importer-api/data-layers/{0}/configure/'.format(upload_layer.id), data=json.dumps(payload), - content_type='application/json') + url = '/importer-api/data-layers/{0}/configure/'.format(upload_layer.id) + response = c.post( + url, data=json.dumps(payload), + content_type='application/json' + ) self.assertTrue(response.status_code, 200) - response = c.get('/importer-api/data-layers/{0}/'.format(upload_layer.id), - content_type='application/json') + url = '/importer-api/data-layers/{0}/'.format(upload_layer.id) + response = c.get(url, content_type='application/json') self.assertTrue(response.status_code, 200) return content @@ -210,8 +212,7 @@ def generic_raster_import(self, file, configuration_options=[{'index': 0}]): return layer def test_multi_upload(self): - """ - Tests Uploading Multiple Files + """Tests Uploading Multiple Files """ upload = self.generic_api_upload( ['boxes_with_year_field.zip', @@ -228,8 +229,7 @@ def test_multi_upload(self): self.assertEqual(9, upload['count']) def test_upload_with_slds(self): - """ - Tests Uploading sld + """Tests Uploading sld """ upload = self.generic_api_upload( ['boxes_with_date.zip', @@ -241,13 +241,13 @@ def test_upload_with_slds(self): ] ) self.assertEqual(6, upload['count']) - uploadid = upload['id'] - uploadobj = UploadedData.objects.get(pk=uploadid) - uplayers = UploadLayer.objects.filter(upload=uploadid) + upload_id = upload['id'] + upload_obj = UploadedData.objects.get(pk=upload_id) + uplayers = UploadLayer.objects.filter(upload=upload_id) layerid = uplayers[0].pk - upfiles_cnt = UploadFile.objects.filter(upload=uploadid).count() - self.assertEqual(6,upfiles_cnt) + upfiles_count = UploadFile.objects.filter(upload=upload_id).count() + self.assertEqual(6,upfiles_count) layer = Layer.objects.get(pk=layerid) gslayer = self.cat.get_layer(layer.name) @@ -256,8 +256,7 @@ def test_upload_with_slds(self): self.assertEqual('boxes.sld',default_style.filename) def test_upload_with_metadata(self): - """ - Tests Uploading metadata + """Tests Uploading metadata """ upload = self.generic_api_upload( ['boxes_with_date.zip', @@ -267,13 +266,13 @@ def test_upload_with_metadata(self): ] ) self.assertEqual(5, upload['count']) - uploadid = upload['id'] - uploadobj = UploadedData.objects.get(pk=uploadid) - uplayers = UploadLayer.objects.filter(upload=uploadid) + upload_id = upload['id'] + upload_obj = UploadedData.objects.get(pk=upload_id) + uplayers = UploadLayer.objects.filter(upload=upload_id) layerid = uplayers[0].pk - upfiles_cnt = UploadFile.objects.filter(upload=uploadid).count() - self.assertEqual(5,upfiles_cnt) + upfiles_count = UploadFile.objects.filter(upload=upload_id).count() + self.assertEqual(5,upfiles_count) layer = Layer.objects.get(pk=layerid) self.assertEqual(layer.language, 'eng') diff --git a/osgeo_importer/views.py b/osgeo_importer/views.py index f0a4f7b..92288ee 100644 --- a/osgeo_importer/views.py +++ b/osgeo_importer/views.py @@ -109,12 +109,16 @@ def form_valid(self, form): for layer in description: configuration_options = DEFAULT_LAYER_CONFIGURATION.copy() configuration_options.update({'index': layer.get('index')}) - upload.uploadlayer_set.add(UploadLayer(upload_file=upfile, - name=upfile_basename, - fields=layer.get('fields', {}), - index=layer.get('index'), - feature_count=layer.get('feature_count', None), - configuration_options=configuration_options)) + upload.uploadlayer_set.add( + UploadLayer( + upload_file=upfile, + name=upfile_basename, + fields=layer.get('fields', {}), + index=layer.get('index'), + feature_count=layer.get('feature_count', None), + configuration_options=configuration_options + ) + ) upload.complete = True upload.state = 'UPLOADED' upload.save() diff --git a/osgeo_importer_prj/settings.py b/osgeo_importer_prj/settings.py index 140c1f7..4ffbb7c 100644 --- a/osgeo_importer_prj/settings.py +++ b/osgeo_importer_prj/settings.py @@ -80,6 +80,9 @@ OSGEO_DATASTORE = 'datastore' OSGEO_IMPORTER_GEONODE_ENABLED = True -OSGEO_IMPORTER_VALID_EXTENSIONS = ['shp','shx','prj','dbf','kml','geojson','json','tif','tiff','gpkg','csv','zip','xml','sld'] +OSGEO_IMPORTER_VALID_EXTENSIONS = [ + 'shp', 'shx', 'prj', 'dbf', 'kml', 'geojson', 'json', 'tif', 'tiff', + 'gpkg', 'csv','zip','xml','sld' +] LOGGING['loggers']['osgeo_importer'] = {"handlers": ["console"], "level": "DEBUG"} DATABASE_ROUTERS = ['osgeo_importer_prj.dbrouters.DefaultOnlyMigrations'] From 109ae4d08394efe90cb42bafdee4a32017afc949 Mon Sep 17 00:00:00 2001 From: Sara Safavi Date: Fri, 23 Sep 2016 17:59:22 -0500 Subject: [PATCH 15/27] Add timestamp to dates header for uploaded files This can help a little to make uploads more distinguishable when they don't have different names. --- osgeo_importer/static/osgeo_importer/partials/upload.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osgeo_importer/static/osgeo_importer/partials/upload.html b/osgeo_importer/static/osgeo_importer/partials/upload.html index 73ec8a2..0a2f6c2 100644 --- a/osgeo_importer/static/osgeo_importer/partials/upload.html +++ b/osgeo_importer/static/osgeo_importer/partials/upload.html @@ -8,8 +8,8 @@
- {{upload.date | date:"MMM d yyyy"}} - {{upload.file_size}} + {{upload.date | date:"MMM d yyyy HH:mm:ss"}} + {{upload.file_size }}
From 838c43597daaa2970d41ebc426755f32b9c1a1dc Mon Sep 17 00:00:00 2001 From: Sara Safavi Date: Fri, 23 Sep 2016 18:23:57 -0500 Subject: [PATCH 16/27] don't reset self.size to zero --- osgeo_importer/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index f3d620b..8621e08 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -154,7 +154,9 @@ def filesize(self): if self.uploadfile_set.count() > 0: for uploadfile in self.uploadfile_set.all(): size += uploadfile.file.size - self.size = size + else: + size = self.uploadfile_set.first().file.size + self.size = size self.save() return sizeof_fmt(self.size) From 6ffb63be9cd740156a47c67c64e0c351f0d6c9ba Mon Sep 17 00:00:00 2001 From: Sasha Hart Date: Fri, 23 Sep 2016 17:53:03 -0500 Subject: [PATCH 17/27] set upload size at ingestion, not first use also permit null file_size for UploadedData. Sometimes an upload doesn't have a size (maybe it didn't finish uploading). In that case, it is not really sensible to say it is 0 bytes. It has no size. --- osgeo_importer/api.py | 2 +- osgeo_importer/models.py | 12 +++--------- osgeo_importer/views.py | 5 +++++ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/osgeo_importer/api.py b/osgeo_importer/api.py index 986be50..df39a71 100644 --- a/osgeo_importer/api.py +++ b/osgeo_importer/api.py @@ -115,7 +115,7 @@ class UploadedDataResource(ModelResource): """ user = ForeignKey(UserResource, 'user') - file_size = CharField(attribute='filesize', readonly=True) + file_size = CharField(attribute='filesize', readonly=True, null=True) layers = ToManyField(UploadedLayerResource, 'uploadlayer_set', full=True) file_url = CharField(attribute='file_url', readonly=True, null=True) diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index 8621e08..6097181 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -149,15 +149,9 @@ def get_delete_url(self): def filesize(self): """Humanizes the upload file size. """ - size = 0 - if not self.size: - if self.uploadfile_set.count() > 0: - for uploadfile in self.uploadfile_set.all(): - size += uploadfile.file.size - else: - size = self.uploadfile_set.first().file.size - self.size = size - self.save() + # whereof one cannot speak, one must be silent + if self.size is None: + return None return sizeof_fmt(self.size) def file_url(self): diff --git a/osgeo_importer/views.py b/osgeo_importer/views.py index 92288ee..87acef4 100644 --- a/osgeo_importer/views.py +++ b/osgeo_importer/views.py @@ -98,8 +98,10 @@ def form_valid(self, form): finalfiles.append(tofile) # Loop through and create uploadfiles and uploadlayers + upfiles = [] for each in finalfiles: upfile = UploadFile.objects.create(upload=upload) + upfiles.append(upfile) upfile.file.name = each upfile.save() upfile_basename = os.path.basename(each) @@ -119,6 +121,9 @@ def form_valid(self, form): configuration_options=configuration_options ) ) + upload.size = sum( + upfile.file.size for upfile in upfiles + ) upload.complete = True upload.state = 'UPLOADED' upload.save() From a72af51d5118bbe7657ca83318f64fef03f5a852 Mon Sep 17 00:00:00 2001 From: Sasha Hart Date: Fri, 23 Sep 2016 18:00:19 -0500 Subject: [PATCH 18/27] set upload name and file type where feasible Since some of the multifile and geopackage changes, it is no longer obvious what should be the name of an upload. If it just contains one file, we can do as before, but otherwise things can become more difficult. The heuristics for this are a little complex, so they've been split off into their own method and tests created just for this method. --- .../osgeo_importer/partials/upload.html | 2 +- osgeo_importer/test_views.py | 81 ++++++++++++++++++ osgeo_importer/views.py | 82 ++++++++++++++++++- 3 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 osgeo_importer/test_views.py diff --git a/osgeo_importer/static/osgeo_importer/partials/upload.html b/osgeo_importer/static/osgeo_importer/partials/upload.html index 0a2f6c2..bb58d44 100644 --- a/osgeo_importer/static/osgeo_importer/partials/upload.html +++ b/osgeo_importer/static/osgeo_importer/partials/upload.html @@ -3,7 +3,7 @@
- {{ upload.name }} + {{ upload.name || "[multiple files]" }}
diff --git a/osgeo_importer/test_views.py b/osgeo_importer/test_views.py new file mode 100644 index 0000000..72eec95 --- /dev/null +++ b/osgeo_importer/test_views.py @@ -0,0 +1,81 @@ +from django.test import TestCase +from osgeo_importer import views +from osgeo_importer import models + + +class TestFileAddView_upload(TestCase): + """Test the helper method FileAddView.upload. + """ + + class FakeRequest(object): + """Fake the kind of request object used by FileAddView.upload. + """ + def __init__(self, user): + self.user = user + + class FakeFile(object): + """Fake the kind of file object used by FileAddView.upload. + """ + def __init__(self, name): + self.name = name + + def view(self): + view = views.FileAddView() + view.get_file_type = lambda path: "BogusType" + user = None + view.request = self.FakeRequest(user) + return view + + def test_empty(self): + """Empty data should return None. + + Form validation should guard against this sort of argument coming to + upload(), but if it does happen, upload() should behave reasonably. + """ + data = [] + view = self.view() + upload = view.upload(data) + self.assertEqual(upload.name, None) + self.assertEqual(upload.file_type, None) + + def test_single(self): + data = [ + self.FakeFile("/tmp/xyz/abc/foo.shp") + ] + view = self.view() + upload = view.upload(data) + self.assertEqual(upload.name, "foo.shp") + self.assertEqual(upload.file_type, "BogusType") + + def test_double(self): + data = [ + self.FakeFile("/tmp/xyz/abc/foo.shp"), + self.FakeFile("/tmp/xyz/abc/bar.shp") + ] + view = self.view() + upload = view.upload(data) + self.assertEqual(upload.name, "bar.shp, foo.shp") + self.assertEqual(upload.file_type, None) + + def test_single_too_long(self): + max_length = models.UploadedData._meta.get_field('name').max_length + too_long = "ObviouslyWayWayWayWayWayWayWayWayWayWayWayWayWayWayTooLong" + self.assertGreater(too_long, max_length) + data = [ + self.FakeFile("/tmp/{0}".format(too_long)) + ] + view = self.view() + upload = view.upload(data) + self.assertEqual(upload.name.startswith("Obviously"), True) + self.assertEqual(upload.file_type, "BogusType") + + def test_many_too_long(self): + data = [ + self.FakeFile("/tmp/xyz/abc/NotTooLongOnItsOwn.shp"), + self.FakeFile("/tmp/xyz/abc/rather_long_to_be_combined_with.shp"), + self.FakeFile("/tmp/really_too_much.shp"), + ] + view = self.view() + upload = view.upload(data) + self.assertEqual(upload.name, None) + self.assertEqual(upload.file_type, None) diff --git a/osgeo_importer/views.py b/osgeo_importer/views.py index 87acef4..9130b23 100644 --- a/osgeo_importer/views.py +++ b/osgeo_importer/views.py @@ -2,9 +2,11 @@ import json import logging import shutil +import collections from django.http import HttpResponse from django.views.generic import FormView, ListView, TemplateView from django.core.urlresolvers import reverse_lazy +from django.utils.text import Truncator from .forms import UploadFileForm from .models import UploadedData, UploadLayer, UploadFile, DEFAULT_LAYER_CONFIGURATION from .importers import OSGEO_IMPORTER @@ -79,8 +81,86 @@ class FileAddView(FormView, ImportHelper, JSONResponseMixin): template_name = 'osgeo_importer/new.html' json = False - def form_valid(self, form): + def upload(self, data): + """Use cleaned form data to populate an unsaved upload record. + + Once, each upload was just one file, so all we had to do was attach its + name and type to the upload, where we could display it. Since multifile + and geopackage supported were added, the required behavior depends on + what we find in the upload. That is what this method is for. For + example, it looks at an upload and tries to come up with a reasonably + mnemonic name, or else set name to None to mean there was no obvious + option. + """ + # Do not save here, we want to leave control of that to the caller. upload = UploadedData.objects.create(user=self.request.user) + # Get a list of paths for files in this upload. + paths = [item.name for item in data] + # If there are multiple paths, see if we can boil them down to a + # smaller number of representative paths (ideally, just one). + if len(paths) > 1: + # Group paths by common pre-extension prefixes + groups = collections.defaultdict(list) + for path in paths: + group_name = os.path.splitext(path)[0] + groups[group_name].append(path) + # Check each group for "leaders" - a special filename like "a.shp" + # which can be understood to represent other files in the group. + # Map from each group name to a list of leaders in that group. + leader_exts = ["shp"] + group_leaders = {} + for group_name, group in groups.items(): + leaders = [ + path for path in group + if any(path.endswith(ext) for ext in leader_exts) + ] + if leaders: + group_leaders[group_name] = leaders + # Rebuild paths: leaders + paths without leaders to represent them + leader_paths = [] + for leaders in group_leaders.values(): + leader_paths.extend(leaders) + orphan_paths = [] + for group_name, group in groups.items(): + if group_name not in group_leaders: + orphan_paths.extend(group) + paths = leader_paths + orphan_paths + + max_length = UploadedData._meta.get_field('name').max_length + # If no data, just return the instance. + if not paths: + name = None + file_type = None + # If we just have one path, that's a reasonable name for the upload. + # We want this case to happen as frequently as reasonable, so that we + # can set one reasonable name and file_type for the whole upload. + elif len(paths) == 1: + path = paths[0] + basename = os.path.basename(path) + name = Truncator(basename).chars(max_length) + file_type = self.get_file_type(path) + # Failing that, see if we can represent all the important paths within + # the available space, making a meaningful mnemonic that isn't + # misleading even though there isn't one obvious name. But if we can't + # even do that, no use generating misleading or useless strings here, + # just pass a None and let the frontend handle the lack of information. + else: + basenames = sorted(os.path.basename(path) for path in paths) + name = ", ".join(basenames) + if len(name) > max_length: + logger.warning( + "rejecting upload name for length: {0!r} {1} > {2}" + .format(name, len(name), max_length) + ) + name = None + file_type = None + + upload.name = name + upload.file_type = file_type + return upload + + def form_valid(self, form): + upload = self.upload(form.cleaned_data['file']) upload.save() # Create Upload Directory based on Upload PK From d68c66bef4f0d4d4c1d52081e36b3a8e2bf91329 Mon Sep 17 00:00:00 2001 From: Sasha Hart Date: Mon, 26 Sep 2016 14:59:48 -0500 Subject: [PATCH 19/27] set layer file_type and present in template No longer possible just to assume that there is one upload file type and it is the same file type as a layer, so we need to detect the actual file type, store it in the database and present it to users. - add to api - add to UploadLayer as property - add to UploadFile as field - add the migration to make the column - set it from the form processing --- osgeo_importer/api.py | 1 + .../migrations/0004_uploadfile_file_type.py | 19 +++++++++++++++++++ osgeo_importer/models.py | 9 +++++++++ .../osgeo_importer/partials/upload.html | 2 +- osgeo_importer/views.py | 8 +++++++- 5 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 osgeo_importer/migrations/0004_uploadfile_file_type.py diff --git a/osgeo_importer/api.py b/osgeo_importer/api.py index df39a71..5ae0d08 100644 --- a/osgeo_importer/api.py +++ b/osgeo_importer/api.py @@ -33,6 +33,7 @@ class UploadedLayerResource(ModelResource): configuration_options = DictField(attribute='configuration_options', null=True) fields = ListField(attribute='fields') status = CharField(attribute='status', readonly=True, null=True) + file_type = CharField(attribute='file_type', readonly=True) class Meta: queryset = UploadLayer.objects.all() diff --git a/osgeo_importer/migrations/0004_uploadfile_file_type.py b/osgeo_importer/migrations/0004_uploadfile_file_type.py new file mode 100644 index 0000000..1361d31 --- /dev/null +++ b/osgeo_importer/migrations/0004_uploadfile_file_type.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('osgeo_importer', '0003_uploadlayer_upload_file'), + ] + + operations = [ + migrations.AddField( + model_name='uploadfile', + name='file_type', + field=models.CharField(max_length=50, null=True, blank=True), + ), + ] diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index 6097181..ca7556b 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -172,6 +172,7 @@ def __unicode__(self): class UploadFile(models.Model): upload = models.ForeignKey(UploadedData, null=True, blank=True) file = models.FileField(upload_to="uploads", validators=[validate_file_extension, validate_inspector_can_read]) + file_type = models.CharField(max_length=50, null=True, blank=True) slug = models.SlugField(max_length=250, blank=True) def __unicode__(self): @@ -205,6 +206,14 @@ class UploadLayer(models.Model): task_id = models.CharField(max_length=36, blank=True, null=True) feature_count = models.IntegerField(null=True, blank=True) + @property + def file_type(self): + """A layer's 'file type' - really the file type of the file it is in. + """ + if not self.upload_file: + return None + return self.upload_file.file_type + @property def layer_data(self): """ diff --git a/osgeo_importer/static/osgeo_importer/partials/upload.html b/osgeo_importer/static/osgeo_importer/partials/upload.html index bb58d44..66f02cf 100644 --- a/osgeo_importer/static/osgeo_importer/partials/upload.html +++ b/osgeo_importer/static/osgeo_importer/partials/upload.html @@ -31,7 +31,7 @@
File Type - {{upload.file_type}} + {{layer.file_type || upload.file_type || "UNKNOWN" }}
Features diff --git a/osgeo_importer/views.py b/osgeo_importer/views.py index 9130b23..7f6f5cc 100644 --- a/osgeo_importer/views.py +++ b/osgeo_importer/views.py @@ -11,7 +11,7 @@ from .models import UploadedData, UploadLayer, UploadFile, DEFAULT_LAYER_CONFIGURATION from .importers import OSGEO_IMPORTER from .inspectors import OSGEO_INSPECTOR -from .utils import import_string +from .utils import import_string, NoDataSourceFound from django.core.files.storage import FileSystemStorage OSGEO_INSPECTOR = import_string(OSGEO_INSPECTOR) @@ -183,6 +183,12 @@ def form_valid(self, form): upfile = UploadFile.objects.create(upload=upload) upfiles.append(upfile) upfile.file.name = each + # Detect and store file type for later reporting, since it is no + # longer true that every upload has only one file type. + try: + upfile.file_type = self.get_file_type(each) + except NoDataSourceFound: + upfile.file_type = None upfile.save() upfile_basename = os.path.basename(each) upfile_root, upfile_ext = os.path.splitext(upfile_basename) From 3568f6a6df190dd09f9c0d7f6a741a96c1919800 Mon Sep 17 00:00:00 2001 From: Sasha Hart Date: Mon, 26 Sep 2016 14:59:55 -0500 Subject: [PATCH 20/27] add file_name to api using UploadLayer property --- osgeo_importer/api.py | 1 + osgeo_importer/models.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/osgeo_importer/api.py b/osgeo_importer/api.py index 5ae0d08..e9e9fb9 100644 --- a/osgeo_importer/api.py +++ b/osgeo_importer/api.py @@ -34,6 +34,7 @@ class UploadedLayerResource(ModelResource): fields = ListField(attribute='fields') status = CharField(attribute='status', readonly=True, null=True) file_type = CharField(attribute='file_type', readonly=True) + file_name = CharField(attribute='file_name', readonly=True) class Meta: queryset = UploadLayer.objects.all() diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index ca7556b..59d4146 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -206,6 +206,12 @@ class UploadLayer(models.Model): task_id = models.CharField(max_length=36, blank=True, null=True) feature_count = models.IntegerField(null=True, blank=True) + @property + def file_name(self): + if not self.upload_file: + return None + return self.upload_file.name + @property def file_type(self): """A layer's 'file type' - really the file type of the file it is in. From b97141edbaeedd282ccb6123c1150f6349969794 Mon Sep 17 00:00:00 2001 From: Sasha Hart Date: Mon, 26 Sep 2016 15:02:05 -0500 Subject: [PATCH 21/27] reformat UploadLayer creation in FileAddView form processing --- osgeo_importer/views.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/osgeo_importer/views.py b/osgeo_importer/views.py index 7f6f5cc..5871c68 100644 --- a/osgeo_importer/views.py +++ b/osgeo_importer/views.py @@ -191,22 +191,22 @@ def form_valid(self, form): upfile.file_type = None upfile.save() upfile_basename = os.path.basename(each) - upfile_root, upfile_ext = os.path.splitext(upfile_basename) + _, upfile_ext = os.path.splitext(upfile_basename) if upfile_ext.lower() not in ['.prj', '.dbf', '.shx']: description = self.get_fields(each) for layer in description: configuration_options = DEFAULT_LAYER_CONFIGURATION.copy() configuration_options.update({'index': layer.get('index')}) - upload.uploadlayer_set.add( - UploadLayer( - upload_file=upfile, - name=upfile_basename, - fields=layer.get('fields', {}), - index=layer.get('index'), - feature_count=layer.get('feature_count', None), - configuration_options=configuration_options - ) ) + upload_layer = UploadLayer( + upload_file=upfile, + name=upfile_basename, + fields=layer.get('fields', {}), + index=layer.get('index'), + feature_count=layer.get('feature_count', None), + configuration_options=configuration_options + ) + upload.uploadlayer_set.add(upload_layer) upload.size = sum( upfile.file.size for upfile in upfiles ) From d9567b36b7c83e5cdc5fe980a8f90ab51199eb84 Mon Sep 17 00:00:00 2001 From: Sasha Hart Date: Mon, 26 Sep 2016 15:00:04 -0500 Subject: [PATCH 22/27] add layer.layer_name to api, UploadLayer, set in form processing, migration to add column. layer.name keeps filename info that is used to import, so we can't stuff actual layer names (e.g. from inside gpkg) there without breaking import. If we change both, any existing databases will be storing a different kind of information in that field, which may be confusing. We can deprecate layer.name if we need to. --- osgeo_importer/api.py | 1 + .../migrations/0005_uploadlayer_layer_name.py | 19 +++++++++++++++++++ osgeo_importer/models.py | 1 + osgeo_importer/tests.py | 1 + osgeo_importer/views.py | 3 +++ 5 files changed, 25 insertions(+) create mode 100644 osgeo_importer/migrations/0005_uploadlayer_layer_name.py diff --git a/osgeo_importer/api.py b/osgeo_importer/api.py index e9e9fb9..499380b 100644 --- a/osgeo_importer/api.py +++ b/osgeo_importer/api.py @@ -35,6 +35,7 @@ class UploadedLayerResource(ModelResource): status = CharField(attribute='status', readonly=True, null=True) file_type = CharField(attribute='file_type', readonly=True) file_name = CharField(attribute='file_name', readonly=True) + layer_name = CharField(attribute='layer_name', readonly=True) class Meta: queryset = UploadLayer.objects.all() diff --git a/osgeo_importer/migrations/0005_uploadlayer_layer_name.py b/osgeo_importer/migrations/0005_uploadlayer_layer_name.py new file mode 100644 index 0000000..162f70f --- /dev/null +++ b/osgeo_importer/migrations/0005_uploadlayer_layer_name.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('osgeo_importer', '0004_uploadfile_file_type'), + ] + + operations = [ + migrations.AddField( + model_name='uploadlayer', + name='layer_name', + field=models.CharField(max_length=64, null=True), + ), + ] diff --git a/osgeo_importer/models.py b/osgeo_importer/models.py index 59d4146..ce5713c 100644 --- a/osgeo_importer/models.py +++ b/osgeo_importer/models.py @@ -205,6 +205,7 @@ class UploadLayer(models.Model): configuration_options = JSONField(null=True) task_id = models.CharField(max_length=36, blank=True, null=True) feature_count = models.IntegerField(null=True, blank=True) + layer_name = models.CharField(max_length=64, null=True) @property def file_name(self): diff --git a/osgeo_importer/tests.py b/osgeo_importer/tests.py index 6c199d5..4a1fde1 100644 --- a/osgeo_importer/tests.py +++ b/osgeo_importer/tests.py @@ -249,6 +249,7 @@ def test_upload_with_slds(self): upfiles_count = UploadFile.objects.filter(upload=upload_id).count() self.assertEqual(6,upfiles_count) + # Warning: this assumes that Layer pks equal UploadLayer pks layer = Layer.objects.get(pk=layerid) gslayer = self.cat.get_layer(layer.name) default_style = gslayer.default_style diff --git a/osgeo_importer/views.py b/osgeo_importer/views.py index 5871c68..a216c46 100644 --- a/osgeo_importer/views.py +++ b/osgeo_importer/views.py @@ -197,10 +197,13 @@ def form_valid(self, form): for layer in description: configuration_options = DEFAULT_LAYER_CONFIGURATION.copy() configuration_options.update({'index': layer.get('index')}) + layer_basename = os.path.basename( + layer.get('layer_name') or '' ) upload_layer = UploadLayer( upload_file=upfile, name=upfile_basename, + layer_name=layer_basename, fields=layer.get('fields', {}), index=layer.get('index'), feature_count=layer.get('feature_count', None), From 428e38f89554e94b24d903e09b5e3ee8f0d25ae1 Mon Sep 17 00:00:00 2001 From: Sasha Hart Date: Mon, 26 Sep 2016 15:05:33 -0500 Subject: [PATCH 23/27] show both layer name and file name in upload list --- osgeo_importer/static/osgeo_importer/partials/upload.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/osgeo_importer/static/osgeo_importer/partials/upload.html b/osgeo_importer/static/osgeo_importer/partials/upload.html index 66f02cf..a53b00a 100644 --- a/osgeo_importer/static/osgeo_importer/partials/upload.html +++ b/osgeo_importer/static/osgeo_importer/partials/upload.html @@ -23,7 +23,9 @@
-
{{layer.name}} +
+ {{layer.layer_name || layer.file_name}} +
(in {{layer.file_name}}) Layer: {{layer.geonode_layer.title}}
From 75e847ab3d20b707361ea46906fe31a1fd838dfa Mon Sep 17 00:00:00 2001 From: Sasha Hart Date: Mon, 26 Sep 2016 15:04:28 -0500 Subject: [PATCH 24/27] don't break alignment from absent layer.status --- osgeo_importer/static/osgeo_importer/partials/upload.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osgeo_importer/static/osgeo_importer/partials/upload.html b/osgeo_importer/static/osgeo_importer/partials/upload.html index a53b00a..dab50ca 100644 --- a/osgeo_importer/static/osgeo_importer/partials/upload.html +++ b/osgeo_importer/static/osgeo_importer/partials/upload.html @@ -41,7 +41,7 @@
Status - {{layer.status}} + {{layer.status || "UNKNOWN" }}
From 8ba6eb173ab6595b47664469ac5073271205d2bb Mon Sep 17 00:00:00 2001 From: Sasha Hart Date: Mon, 26 Sep 2016 15:04:49 -0500 Subject: [PATCH 25/27] don't show feature count for rasters where it's meaningless --- osgeo_importer/static/osgeo_importer/partials/upload.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osgeo_importer/static/osgeo_importer/partials/upload.html b/osgeo_importer/static/osgeo_importer/partials/upload.html index dab50ca..8ff14c7 100644 --- a/osgeo_importer/static/osgeo_importer/partials/upload.html +++ b/osgeo_importer/static/osgeo_importer/partials/upload.html @@ -35,7 +35,7 @@ File Type {{layer.file_type || upload.file_type || "UNKNOWN" }}
-
+
Features {{layer.feature_count | number : 0}}
From fd40e26ebe4a398c966ce3605e8b02e7ec5eb10c Mon Sep 17 00:00:00 2001 From: Sara Safavi Date: Mon, 26 Sep 2016 15:56:36 -0500 Subject: [PATCH 26/27] Move default list of supported extensions into its on var --- osgeo_importer/importers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/osgeo_importer/importers.py b/osgeo_importer/importers.py index 0449cab..ee87818 100644 --- a/osgeo_importer/importers.py +++ b/osgeo_importer/importers.py @@ -25,9 +25,10 @@ MEDIA_ROOT = getattr(settings, 'MEDIA_ROOT', FileSystemStorage().location) OSGEO_IMPORTER = getattr(settings, 'OSGEO_IMPORTER', 'osgeo_importer.importers.OGRImport') -VALID_EXTENSIONS = getattr(settings, 'OSGEO_IMPORTER_VALID_EXTENSIONS', - ['shp', 'shx', 'prj', 'dbf', 'kml', 'geojson', 'json', - 'tif', 'tiff', 'gpkg', 'csv', 'zip', 'xml', 'sld']) +DEFAULT_SUPPORTED_EXTENSIONS = ['shp', 'shx', 'prj', 'dbf', 'kml', 'geojson', 'json', + 'tif', 'tiff', 'gpkg', 'csv', 'zip', 'xml', 'sld'] +VALID_EXTENSIONS = getattr(settings, 'OSGEO_IMPORTER_VALID_EXTENSIONS', DEFAULT_SUPPORTED_EXTENSIONS) + RASTER_FILES = getattr(settings, 'OSGEO_IMPORTER_RASTER_FILES', os.path.join(MEDIA_ROOT, 'osgeo_importer_raster')) UPLOAD_DIR = getattr(settings, 'OSGEO_IMPORTER_UPLOAD_DIR', os.path.join(MEDIA_ROOT, 'osgeo_importer_uploads')) From 39522d2e31d3b80df4664d25d64377e145ac700d Mon Sep 17 00:00:00 2001 From: Sara Safavi Date: Mon, 26 Sep 2016 15:58:45 -0500 Subject: [PATCH 27/27] Use list of valid extensions to populate text in the upload template Whether extensions are explicitly specified in settings, or else just the default list of extensions is used, we should consider importers.VALID_EXTENSIONS as canon and use that in user-facing templates --- osgeo_importer/templates/osgeo_importer/new.html | 3 +++ osgeo_importer/views.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/osgeo_importer/templates/osgeo_importer/new.html b/osgeo_importer/templates/osgeo_importer/new.html index bab7188..330f48f 100644 --- a/osgeo_importer/templates/osgeo_importer/new.html +++ b/osgeo_importer/templates/osgeo_importer/new.html @@ -39,6 +39,9 @@

{% trans "Add data" %}

{% for error in form.file.errors %} {{ error|escape }} {% endfor %} +
+ Valid file extensions: {{ VALID_EXTENSIONS }} +
diff --git a/osgeo_importer/views.py b/osgeo_importer/views.py index a216c46..1dad195 100644 --- a/osgeo_importer/views.py +++ b/osgeo_importer/views.py @@ -9,7 +9,7 @@ from django.utils.text import Truncator from .forms import UploadFileForm from .models import UploadedData, UploadLayer, UploadFile, DEFAULT_LAYER_CONFIGURATION -from .importers import OSGEO_IMPORTER +from .importers import OSGEO_IMPORTER, VALID_EXTENSIONS from .inspectors import OSGEO_INSPECTOR from .utils import import_string, NoDataSourceFound from django.core.files.storage import FileSystemStorage @@ -225,6 +225,9 @@ def form_valid(self, form): def render_to_response(self, context, **response_kwargs): + # grab list of valid importer extensions for use in templates + context["VALID_EXTENSIONS"] = ", ".join(VALID_EXTENSIONS) + if self.json: context = {'errors': context['form'].errors} return self.render_to_json_response(context, **response_kwargs)