Update Travis Setup

.travis.yml

@@ -1,24 +1,23 @@
-dist: trusty
-
 os: linux
+dist: focal
 language: python
 
-install:
- - bash bin/travis-build.bash
 services:
+ - docker
  - redis
- - postgresql
 
+install: bash bin/travis-build.bash
 script: bash bin/travis-run.bash
-before_install:
- - pip install codecov
-after_success:
- - codecov
+
+stages:
+ - Flake8
+ - Tests
+
 
 jobs:
  include:
  - stage: Flake8
- python: 2.7
+ python: 3.6
  env: FLAKE8=True
  install:
  - pip install flake8==3.5.0
@@ -28,17 +27,45 @@ jobs:
  # stop the build if there are Python syntax errors or undefined names
  - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
  # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
- # - flake8 . --count --max-line-length=127 --statistics --exclude ckan --exit-zero
+ - flake8 . --count --max-line-length=127 --statistics --exclude ckan --exit-zero
+
+
  - stage: Tests
- python: "2.7"
- env: CKANVERSION=master
- - python: "3.6"
- env: CKANVERSION=master
+ python: "3.6"
+ env: CKANVERSION=2.9
+ services:
+ - postgresql
+ - redis
+ - docker
+
+
+ - python: "2.7"
+ env: CKANVERSION=2.9
+ services:
+ - postgresql
+ - redis
+ - docker
+
  - python: "2.7"
  env: CKANVERSION=2.8
+ addons:
+ postgresql: '11'
+ apt:
+ sources:
+ - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main'
+ packages:
+ - postgresql-11
+
  - python: "2.7"
  env: CKANVERSION=2.7
-
+ addons:
+ postgresql: '9.6'
+ apt:
+ sources:
+ - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main'
+ packages:
+ - postgresql-9.6
+
 cache:
  directories:
- - $HOME/.cache/pip
+ - $HOME/.cache/pip

bin/travis-build.bash

@@ -2,10 +2,15 @@
 set -e
 
 echo "This is travis-build.bash..."
+echo "Targetting CKAN $CKANVERSION on Python $TRAVIS_PYTHON_VERSION"
+if [ $CKANVERSION == 'master' ]
+then
+ export CKAN_MINOR_VERSION=100
+else
+ export CKAN_MINOR_VERSION=${CKANVERSION##*.}
+fi
 
-echo "Installing the packages that CKAN requires..."
-sudo apt-get update -qq
-sudo apt-get install solr-jetty
+export PYTHON_MAJOR_VERSION=${TRAVIS_PYTHON_VERSION%.*}
 
 echo "Installing CKAN and its Python dependencies..."
 git clone https://github.com/ckan/ckan
@@ -19,67 +24,79 @@ else
  echo "CKAN version: ${CKAN_TAG#ckan-}"
 fi
 
-# install the recommended version of setuptools
+echo "Installing the recommended setuptools requirement"
 if [ -f requirement-setuptools.txt ]
 then
- echo "Updating setuptools..."
  pip install -r requirement-setuptools.txt
 fi
 
-if [ $CKANVERSION == '2.7' ]
-then
- echo "Installing setuptools"
- pip install setuptools==39.0.1
-fi
-
 python setup.py develop
-if [ -f requirements-py2.txt ]
+
+if (( $CKAN_MINOR_VERSION >= 9 )) && (( $PYTHON_MAJOR_VERSION == 2 ))
 then
  pip install -r requirements-py2.txt
 else
- # To avoid error:
- # Error: could not determine PostgreSQL version from '10.1'
- # we need newer psycopg2 and corresponding exc name change
- sed -i -e 's/psycopg2==2.4.5/psycopg2==2.8.2/' requirements.txt
- sed -i -e 's/except sqlalchemy.exc.InternalError:/except (sqlalchemy.exc.InternalError, sqlalchemy.exc.DBAPIError):/' ckan/config/environment.py
  pip install -r requirements.txt
 fi
+
 pip install -r dev-requirements.txt
 cd -
 
+echo "Setting up Solr..."
+docker run --name ckan-solr -p 8983:8983 -d openknowledge/ckan-solr-dev:$CKANVERSION
+
+echo "Setting up Postgres..."
+export PG_VERSION="$(pg_lsclusters | grep online | awk '{print $1}')"
+export PG_PORT="$(pg_lsclusters | grep online | awk '{print $3}')"
+echo "Using Postgres $PGVERSION on port $PG_PORT"
+if [ $PG_PORT != "5432" ]
+then
+ echo "Using non-standard Postgres port, updating configuration..."
+ sed -i -e "s/postgresql:\/\/ckan_default:pass@localhost\/ckan_test/postgresql:\/\/ckan_default:pass@localhost:$PG_PORT\/ckan_test/" ckan/test-core.ini
+ sed -i -e "s/postgresql:\/\/ckan_default:pass@localhost\/datastore_test/postgresql:\/\/ckan_default:pass@localhost:$PG_PORT\/datastore_test/" ckan/test-core.ini
+ sed -i -e "s/postgresql:\/\/datastore_default:pass@localhost\/datastore_test/postgresql:\/\/datastore_default:pass@localhost:$PG_PORT\/datastore_test/" ckan/test-core.ini
+fi
+
+
 echo "Creating the PostgreSQL user and database..."
-sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
-sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
-sudo -u postgres psql -c "CREATE USER datastore_default WITH PASSWORD 'pass';"
-sudo -u postgres psql -c 'CREATE DATABASE datastore_test WITH OWNER datastore_default;'
+sudo -u postgres psql -p $PG_PORT -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
+sudo -u postgres psql -p $PG_PORT -c "CREATE USER datastore_default WITH PASSWORD 'pass';"
+sudo -u postgres psql -p $PG_PORT -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
+sudo -u postgres psql -p $PG_PORT -c 'CREATE DATABASE datastore_test WITH OWNER ckan_default;'
+
 
-echo "Setting up Solr..."
-# Solr is multicore for tests on ckan master, but it's easier to run tests on
-# Travis single-core. See https://github.com/ckan/ckan/issues/2972
-sed -i -e 's/solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' ckan/test-core.ini
-printf "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
-sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
-sudo service jetty restart
 
 echo "Create full text function..."
 cp full_text_function.sql /tmp
 cd /tmp
-sudo -u postgres psql datastore_test -f full_text_function.sql
+echo "Just BEFORE running full text function..."
+echo "checking out sockets on /var/run/postgresql/..."
+ls -la /var/run/postgresql/
+sudo -u postgres psql -p $PG_PORT datastore_test -f full_text_function.sql
+echo "Just AFTER running full text function...."
 cd -
 
 echo "Initialising the database..."
 cd ckan
-paster db init -c test-core.ini
-paster datastore set-permissions -c test-core.ini | sudo -u postgres psql
+if (( $CKAN_MINOR_VERSION >= 9 ))
+then
+ ckan -c test-core.ini db init
+ ckan -c test-core.ini datastore set-permissions | sudo -u postgres psql -p $PG_PORT
+else
+ paster db init -c test-core.ini
+ paster datastore set-permissions -c test-core.ini | sudo -u postgres psql -p $PG_PORT
+fi
 cd -
 
 echo "Installing ckanext-xloader and its requirements..."
-python setup.py develop
+pip install -r pip-requirements.txt
 pip install -r requirements.txt
 pip install -r dev-requirements.txt
 
-echo "Moving test.ini into a subdir..."
+python setup.py develop
+
+echo "Moving test.ini into a subdir... (because the core ini file is referenced as ../ckan/test-core.ini)"
 mkdir subdir
 mv test.ini subdir
 
-echo "travis-build.bash is done."
+echo "The travis-build.bash is done."

bin/travis-run.bash

@@ -1,11 +1,5 @@
-#!/bin/sh -e
+#!/bin/bash
 set -ex
 
-flake8 --version
-# stop the build if there are Python syntax errors or undefined names
-flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan,ckanext-xloader
-
-pytest --ckan-ini subdir/test.ini --cov=ckanext.xloader ckanext/xloader/tests
-
-# strict linting
-flake8 . --count --max-complexity=27 --max-line-length=127 --statistics --exclude ckan,ckanext-xloader
+pip install future
+pytest --ckan-ini=subdir/test.ini --cov=ckanext.xloader --disable-warnings ckanext/xloader/tests

ckanext/xloader/loader.py

@@ -227,7 +227,7 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None):
  # e is a str but with foreign chars e.g.
  # 'extra data: "paul,pa\xc3\xbcl"\n'
  # but logging and exceptions need a normal (7 bit) str
- error_str = str(e).decode('ascii', 'replace').encode('ascii', 'replace')
+ error_str = str(e)
  logger.warning(error_str)
  raise LoaderError('Error during the load into PostgreSQL:'
  ' {}'.format(error_str))

pip-requirements.txt

@@ -0,0 +1,8 @@
+ckantoolkit==0.0.3
+pika>=1.1.0
+pyOpenSSL==18.0.0
+redis
+requests>=2.11.1
+six>=1.12.0
+messytables==0.15.2
+Unidecode==1.0.22