Python 3 support

.travis.yml

@@ -3,24 +3,49 @@ dist: trusty
 group: deprecated-2017Q4
 
 language: python
-sudo: required
-python:
-    - "2.7"
-env:
-    - CKANVERSION=master
-    - CKANVERSION=2.4
-    - CKANVERSION=2.5
-    - CKANVERSION=2.6
-    - CKANVERSION=2.7
-    - CKANVERSION=2.8
 
 install:
     - bash bin/travis-build.bash
 services:
-    - redis-server
+    - redis
     - postgresql
-script: sh bin/travis-run.sh
+
+script: bash bin/travis-run.bash
 before_install:
     - pip install codecov
 after_success:
     - codecov
+
+jobs:
+  include:
+    - stage: Flake8
+      python: 2.7
+      env: FLAKE8=True
+      install:
+        - pip install flake8==3.5.0
+        - pip install pycodestyle==2.3.0
+      script:
+        - flake8 --version
+        # stop the build if there are Python syntax errors or undefined names
+        - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+        # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
+        - flake8 . --count --max-line-length=127 --statistics --exclude ckan --exit-zero
+    - stage: Tests
+      python: "2.7"
+      env: CKANVERSION=master
+    - python: "3.6"
+      env: CKANVERSION=master
+    - python: "2.7"
+      env: CKANVERSION=2.8
+    - python: "2.7"
+      env: CKANVERSION=2.7
+    - python: "2.7"
+      env: CKANVERSION=2.6
+    - python: "2.7"
+      env: CKANVERSION=2.5
+    - python: "2.7"
+      env: CKANVERSION=2.4
+
+cache:
+  directories:
+    - $HOME/.cache/pip

README.md

@@ -910,9 +910,14 @@ Example output of structured data in JSON-LD:
 
 ## Running the Tests
 
-To run the tests, do:
+To run the tests on CKAN >= 2.9, do:
 
-    nosetests --nologcapture --ckan --with-pylons=test.ini ckanext
+    pytest --ckan-ini=test.ini ckanext/dcat/tests
+
+
+To run the tests on CKAN <= 2.8, do:
+
+    nosetests --nologcapture --ckan --with-pylons=test-nose.ini ckanext/dcat/tests/nose
 
 ## Releases
 

bin/travis-build.bash

@@ -1,7 +1,17 @@
-    #nex!/bin/bash
+#!/bin/bash
 set -e
 
 echo "This is travis-build.bash..."
+echo "Targetting CKAN $CKANVERSION on Python $TRAVIS_PYTHON_VERSION"
+if [ $CKANVERSION == 'master' ]
+then
+    export CKAN_MINOR_VERSION=100
+else
+    export CKAN_MINOR_VERSION=${CKANVERSION##*.}
+fi
+
+export PYTHON_MAJOR_VERSION=${TRAVIS_PYTHON_VERSION%.*}
+
 
 echo "Installing the packages that CKAN requires..."
 sudo apt-get update -qq
@@ -39,7 +49,7 @@ then
     pip install setuptools==39.0.1
 fi
 
-if [ -f requirements-py2.txt ]
+if (( $CKAN_MINOR_VERSION >= 9 )) && (( $PYTHON_MAJOR_VERSION == 2 ))
 then
     pip install -r requirements-py2.txt
 else
@@ -59,15 +69,28 @@ sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
 
 echo "Initialising the database..."
 cd ckan
-paster db init -c test-core.ini
+
+
+if (( $CKAN_MINOR_VERSION >= 9 ))
+then
+    ckan -c test-core.ini db init
+else
+    paster db init -c test-core.ini
+fi
 cd -
 
 echo "Installing ckanext-harvest and its requirements..."
 git clone https://github.com/ckan/ckanext-harvest
 cd ckanext-harvest
 python setup.py develop
 pip install -r pip-requirements.txt
-paster harvester initdb -c ../ckan/test-core.ini
+
+if (( $CKAN_MINOR_VERSION >= 9 ))
+then
+    ckan -c test.ini harvester initdb
+else
+    paster harvester initdb -c test.ini
+fi
 cd -
 
 echo "Installing ckanext-dcat and its requirements..."
@@ -79,5 +102,6 @@ python setup.py develop
 echo "Moving test.ini into a subdir..."
 mkdir subdir
 mv test.ini subdir
+mv test-nose.ini subdir
 
 echo "travis-build.bash is done."

bin/travis-run.bash

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+if [ $CKANVERSION == 'master' ]
+then
+    export CKAN_MINOR_VERSION=100
+else
+    export CKAN_MINOR_VERSION=${CKANVERSION##*.}
+fi
+
+
+if (( $CKAN_MINOR_VERSION >= 9 ))
+then
+    pytest --ckan-ini=subdir/test.ini --cov=ckanext.dcat ckanext/dcat/tests
+else
+    nosetests --ckan --nologcapture --with-pylons=subdir/test-nose.ini --with-coverage --cover-package=ckanext.dcat --cover-inclusive --cover-erase --cover-tests ckanext/dcat/tests/nose
+fi

ckanext/dcat/blueprints.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+from flask import Blueprint, jsonify, make_response
+
+from ckantoolkit import config
+
+from ckan.views.dataset import CreateView
+
+import ckan.plugins.toolkit as toolkit
+import ckanext.dcat.utils as utils
+
+dcat = Blueprint(
+    'dcat',
+    __name__,
+    url_defaults={u'package_type': u'dataset'}
+)
+
+
+def read_catalog(_format=None, package_type=None):
+    return utils.read_catalog_page(_format)
+
+
+def read_dataset(_id, _format=None, package_type=None):
+    return utils.read_dataset_page(_id, _format)
+
+if toolkit.asbool(config.get(utils.ENABLE_RDF_ENDPOINTS_CONFIG, True)):
+
+    # requirements={'_format': 'xml|rdf|n3|ttl|jsonld'}
+    dcat.add_url_rule(config.get('ckanext.dcat.catalog_endpoint',
+                                 utils.DEFAULT_CATALOG_ENDPOINT).replace(
+                                     '{_format}', '<_format>'),
+                      view_func=read_catalog)
+    dcat.add_url_rule('/dataset/<_id>.<_format>', view_func=read_dataset)
+
+if toolkit.asbool(config.get(utils.ENABLE_CONTENT_NEGOTIATION_CONFIG)):
+    dcat.add_url_rule('/', view_func=read_catalog)
+
+    dcat.add_url_rule('/dataset/new', view_func=CreateView.as_view(str(u'new')))
+    dcat.add_url_rule('/dataset/<_id>', view_func=read_dataset)
+
+dcat_json_interface = Blueprint('dcat_json_interface', __name__)
+
+
+def dcat_json():
+    datasets = utils.dcat_json_page()
+    return jsonify(datasets)
+
+
+dcat_json_interface.add_url_rule(config.get('ckanext.dcat.json_endpoint',
+                                            '/dcat.json'),
+                                 view_func=dcat_json)

ckanext/dcat/cli.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+import click
+import ckan.plugins.toolkit as tk
+import ckanext.dcat.utils as utils
+
+@click.group()
+def generate_static():
+    """Generates static files containing all datasets.
+
+    """
+    pass
+
+@generate_static.command()
+@click.argument('output', type=click.File(mode="w"))
+def json(output):
+    """The generate command will generate a static file containing all of
+    the datasets in the catalog in JSON format.
+
+    """
+    utils.generate_static_json(output)
+
+
+def get_commands():
+    return [generate_static]

ckanext/dcat/commands.py

@@ -1,9 +1,10 @@
-import json
-import logging
+# -*- coding: utf-8 -*-
 
-from pylons import config
+import logging
 from ckan import plugins as p
 
+import ckanext.dcat.utils as utils
+
 
 class GenerateStaticDCATCommand(p.toolkit.CkanCommand):
     """
@@ -42,25 +43,5 @@ def generate(self, output):
         Keep reading and converting datasets until we get an empty list back
         from dcat_datasets_list
         """
-        data_dict = {'page': 0}
-
         with open(output, 'w') as f:
-            f.write(u"[")
-
-            while True:
-                try:
-                    data_dict['page'] = data_dict['page'] + 1
-                    datasets = \
-                        p.toolkit.get_action('dcat_datasets_list')({},
-                                                                   data_dict)
-                except p.toolkit.ValidationError, e:
-                    self.log.exception(e)
-                    break
-
-                if not datasets:
-                    break
-
-                for dataset in datasets:
-                    f.write(json.dumps(dataset))
-
-            f.write(u"]")
+            utils.generate_static_json(f)