Supports Python3

.github/workflows/ci.yml

@@ -0,0 +1,103 @@
+name: ckanext-spatial ci
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        ckan-version: [2.9, 2.9-py2, 2.8, 2.7]
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }}
+    runs-on: ubuntu-latest
+    container:
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+    services:
+      solr:
+        image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
+      postgres:
+        image: postgis/postgis:10-3.1
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+        image: redis:3
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+      PGPASSWORD: postgres
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Create Database
+      run: |
+        psql --host=postgres --username=postgres --command="CREATE USER ckan_default WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=ckan_default ckan_test
+        psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+        psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+        createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test
+    - name: Install harvester
+      run: |
+        git clone https://github.com/ckan/ckanext-harvest
+        cd ckanext-harvest
+        pip install -r pip-requirements.txt
+        pip install -r dev-requirements.txt
+        pip install -e .
+    - name: Install dependency (common)
+      run: |
+        apk add --no-cache \
+          geos \
+          geos-dev \
+          proj-util \
+          proj-dev \
+          libxml2 \
+          libxslt \
+          gcc \
+          libxml2-dev \
+          libxslt-dev
+    - name: Install dependency (python2)
+      if: ${{ matrix.ckan-version != '2.9' }}
+      run: |
+        apk add --no-cache \
+          python2-dev
+    - name: Install dependency (python3)
+      if: ${{ matrix.ckan-version == '2.9' }}
+      run: |
+        apk add --no-cache \
+          python3-dev
+    - name: Install requirements (python2)
+      if: ${{ matrix.ckan-version != '2.9' }}
+      run: |
+        pip install -r pip-requirements.txt
+    - name: Install requirements (python3)
+      if: ${{ matrix.ckan-version == '2.9' }}
+      run: |
+        pip install -r pip3-requirements.txt
+    - name: Install requirements
+      run: |
+        pip install -e .
+        # Replace default path to CKAN core config file with the one on the container
+        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+    - name: setup postgis
+      run: |
+        psql --host=postgres --username=postgres -d ckan_test --command="ALTER ROLE ckan_default WITH superuser;"
+        psql --host=postgres --username=postgres -d ckan_test --command="CREATE EXTENSION postgis;"
+    - name: Run tests
+      run: pytest --ckan-ini=test.ini ckanext/spatial/tests

bin/ckan_pycsw.py

@@ -75,7 +75,7 @@ def load(pycsw_config, ckan_url):
         response = requests.get(url)
         listing = response.json()
         if not isinstance(listing, dict):
-            raise RuntimeError, 'Wrong API response: %s' % listing
+            raise RuntimeError('Wrong API response: %s' % listing)
         results = listing.get('results')
         if not results:
             break
@@ -115,7 +115,7 @@ def load(pycsw_config, ckan_url):
             ckan_id=ckan_id).delete()
             log.info('Deleted %s' % ckan_id)
             repo.session.commit()
-        except Exception, err:
+        except Exception as err:
             repo.session.rollback()
             raise
 
@@ -128,7 +128,7 @@ def load(pycsw_config, ckan_url):
         try:
             repo.insert(record, 'local', util.get_today_and_now())
             log.info('Inserted %s' % ckan_id)
-        except Exception, err:
+        except Exception as err:
             log.error('ERROR: not inserted %s Error:%s' % (ckan_id, err))
 
     for ckan_id in changed:
@@ -145,9 +145,9 @@ def load(pycsw_config, ckan_url):
             ckan_id=ckan_id).update(update_dict)
             repo.session.commit()
             log.info('Changed %s' % ckan_id)
-        except Exception, err:
+        except Exception as err:
             repo.session.rollback()
-            raise RuntimeError, 'ERROR: %s' % str(err)
+            raise RuntimeError('ERROR: %s' % str(err))
 
 
 def clear(pycsw_config):
@@ -174,13 +174,13 @@ def get_record(context, repo, ckan_url, ckan_id, ckan_info):
 
     try:
         xml = etree.parse(io.BytesIO(response.content))
-    except Exception, err:
+    except Exception as err:
         log.error('Could not pass xml doc from %s, Error: %s' % (ckan_id, err))
         return
 
     try:
         record = metadata.parse_record(context, xml, repo)[0]
-    except Exception, err:
+    except Exception as err:
         log.error('Could not extract metadata from %s, Error: %s' % (ckan_id, err))
         return
 
@@ -270,5 +270,5 @@ def _load_config(file_path):
     elif arg.command == 'clear':
         clear(pycsw_config)
     else:
-        print 'Unknown command {0}'.format(arg.command)
+        print('Unknown command {0}'.format(arg.command))
         sys.exit(1)

ckanext/spatial/cli.py

@@ -0,0 +1,53 @@
+# encoding: utf-8
+import click
+import logging
+
+import ckanext.spatial.util as util
+
+
+log = logging.getLogger(__name__)
+
+def get_commands():
+    return [
+        spatial,
+        spatial_validation
+    ]
+
+
+@click.group(u"spatial-validation", short_help=u"Validation commands")
+def spatial_validation():
+    pass
+
+
+@spatial_validation.command()
+@click.argument('pkg', required=False)
+def report(pkg):
+    return util.report(pkg)
+
+
+@spatial_validation.command('report-csv')
+@click.argument('filepath')
+def report_csv(filepath):
+    return util.report_csv(filepath)
+
+
+@spatial_validation.command('file')
+@click.argument('filepath')
+def validate_file(filepath):
+    return util.validate_file(filepath)
+
+
+@click.group(short_help=u"Performs spatially related operations.")
+def spatial():
+    pass
+
+
+@spatial.command()
+@click.argument('srid', required=False)
+def initdb(srid):
+    return util.initdb(srid)
+
+
+@spatial.command('extents')
+def update_extents():
+    return util.update_extents()

ckanext/spatial/commands/csw.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import sys
 import logging
 
@@ -63,4 +64,4 @@ def command(self):
         elif cmd == 'clear':
             ckan_pycsw.clear(config)
         else:
-            print 'Command %s not recognized' % cmd
+            print('Command %s not recognized' % cmd)