From 75e9bb6b200c03b7de1a5d0d82523be4e9c1d3de Mon Sep 17 00:00:00 2001 From: Brett <54408245+kowh-ai@users.noreply.github.com> Date: Tue, 13 Oct 2020 11:19:41 +0200 Subject: [PATCH 1/6] Update Travis Setup --- .travis.yml | 99 +++++++++++++++++++++++++++---------------- bin/travis-build.bash | 85 ++++++++++++++++++++++--------------- bin/travis-run.bash | 12 ++---- 3 files changed, 117 insertions(+), 79 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f109b13..18c9fc9a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,44 +1,71 @@ -dist: trusty - os: linux +dist: focal language: python -install: - - bash bin/travis-build.bash services: - - redis - - postgresql + - docker + - redis +install: bash bin/travis-build.bash script: bash bin/travis-run.bash -before_install: - - pip install codecov -after_success: - - codecov -jobs: - include: - - stage: Flake8 - python: 2.7 - env: FLAKE8=True - install: - - pip install flake8==3.5.0 - - pip install pycodestyle==2.3.0 - script: - - flake8 --version - # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - # - flake8 . --count --max-line-length=127 --statistics --exclude ckan --exit-zero - - stage: Tests - python: "2.7" - env: CKANVERSION=master - - python: "3.6" - env: CKANVERSION=master - - python: "2.7" - env: CKANVERSION=2.8 - - python: "2.7" - env: CKANVERSION=2.7 +stages: + - Flake8 + - Tests + -cache: - directories: - - $HOME/.cache/pip + jobs: + include: + - stage: Flake8 + python: 3.6 + env: FLAKE8=True + install: + - pip install flake8==3.5.0 + - pip install pycodestyle==2.3.0 + script: + - flake8 --version + # stop the build if there are Python syntax errors or undefined names + - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + - flake8 . --count --max-line-length=127 --statistics --exclude ckan --exit-zero + + + - stage: Tests + python: "3.6" + env: CKANVERSION=2.9 + services: + - postgresql + - redis + - docker + + + - python: "2.7" + env: CKANVERSION=2.9 + services: + - postgresql + - redis + - docker + + - python: "2.7" + env: CKANVERSION=2.8 + addons: + postgresql: '11' + apt: + sources: + - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main' + packages: + - postgresql-11 + + - python: "2.7" + env: CKANVERSION=2.7 + addons: + postgresql: '9.6' + apt: + sources: + - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main' + packages: + - postgresql-9.6 + + cache: + directories: + - $HOME/.cache/pip \ No newline at end of file diff --git a/bin/travis-build.bash b/bin/travis-build.bash index 862efa8e..fc66cf6e 100644 --- a/bin/travis-build.bash +++ b/bin/travis-build.bash @@ -2,10 +2,15 @@ set -e echo "This is travis-build.bash..." +echo "Targetting CKAN $CKANVERSION on Python $TRAVIS_PYTHON_VERSION" +if [ $CKANVERSION == 'master' ] +then + export CKAN_MINOR_VERSION=100 +else + export CKAN_MINOR_VERSION=${CKANVERSION##*.} +fi -echo "Installing the packages that CKAN requires..." -sudo apt-get update -qq -sudo apt-get install solr-jetty +export PYTHON_MAJOR_VERSION=${TRAVIS_PYTHON_VERSION%.*} echo "Installing CKAN and its Python dependencies..." git clone https://github.com/ckan/ckan @@ -19,67 +24,79 @@ else echo "CKAN version: ${CKAN_TAG#ckan-}" fi -# install the recommended version of setuptools +echo "Installing the recommended setuptools requirement" if [ -f requirement-setuptools.txt ] then - echo "Updating setuptools..." pip install -r requirement-setuptools.txt fi -if [ $CKANVERSION == '2.7' ] -then - echo "Installing setuptools" - pip install setuptools==39.0.1 -fi - python setup.py develop -if [ -f requirements-py2.txt ] + +if (( $CKAN_MINOR_VERSION >= 9 )) && (( $PYTHON_MAJOR_VERSION == 2 )) then pip install -r requirements-py2.txt else - # To avoid error: - # Error: could not determine PostgreSQL version from '10.1' - # we need newer psycopg2 and corresponding exc name change - sed -i -e 's/psycopg2==2.4.5/psycopg2==2.8.2/' requirements.txt - sed -i -e 's/except sqlalchemy.exc.InternalError:/except (sqlalchemy.exc.InternalError, sqlalchemy.exc.DBAPIError):/' ckan/config/environment.py pip install -r requirements.txt fi + pip install -r dev-requirements.txt cd - +echo "Setting up Solr..." +docker run --name ckan-solr -p 8983:8983 -d openknowledge/ckan-solr-dev:$CKANVERSION + +echo "Setting up Postgres..." +export PG_VERSION="$(pg_lsclusters | grep online | awk '{print $1}')" +export PG_PORT="$(pg_lsclusters | grep online | awk '{print $3}')" +echo "Using Postgres $PGVERSION on port $PG_PORT" +if [ $PG_PORT != "5432" ] +then + echo "Using non-standard Postgres port, updating configuration..." + sed -i -e "s/postgresql:\/\/ckan_default:pass@localhost\/ckan_test/postgresql:\/\/ckan_default:pass@localhost:$PG_PORT\/ckan_test/" ckan/test-core.ini + sed -i -e "s/postgresql:\/\/ckan_default:pass@localhost\/datastore_test/postgresql:\/\/ckan_default:pass@localhost:$PG_PORT\/datastore_test/" ckan/test-core.ini + sed -i -e "s/postgresql:\/\/datastore_default:pass@localhost\/datastore_test/postgresql:\/\/datastore_default:pass@localhost:$PG_PORT\/datastore_test/" ckan/test-core.ini +fi + + echo "Creating the PostgreSQL user and database..." -sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';" -sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;' -sudo -u postgres psql -c "CREATE USER datastore_default WITH PASSWORD 'pass';" -sudo -u postgres psql -c 'CREATE DATABASE datastore_test WITH OWNER datastore_default;' +sudo -u postgres psql -p $PG_PORT -c "CREATE USER ckan_default WITH PASSWORD 'pass';" +sudo -u postgres psql -p $PG_PORT -c "CREATE USER datastore_default WITH PASSWORD 'pass';" +sudo -u postgres psql -p $PG_PORT -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;' +sudo -u postgres psql -p $PG_PORT -c 'CREATE DATABASE datastore_test WITH OWNER ckan_default;' + -echo "Setting up Solr..." -# Solr is multicore for tests on ckan master, but it's easier to run tests on -# Travis single-core. See https://github.com/ckan/ckan/issues/2972 -sed -i -e 's/solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' ckan/test-core.ini -printf "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty -sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml -sudo service jetty restart echo "Create full text function..." cp full_text_function.sql /tmp cd /tmp -sudo -u postgres psql datastore_test -f full_text_function.sql +echo "Just BEFORE running full text function..." +echo "checking out sockets on /var/run/postgresql/..." +ls -la /var/run/postgresql/ +sudo -u postgres psql -p $PG_PORT datastore_test -f full_text_function.sql +echo "Just AFTER running full text function...." cd - echo "Initialising the database..." cd ckan -paster db init -c test-core.ini -paster datastore set-permissions -c test-core.ini | sudo -u postgres psql +if (( $CKAN_MINOR_VERSION >= 9 )) +then + ckan -c test-core.ini db init + ckan -c test-core.ini datastore set-permissions | sudo -u postgres psql -p $PG_PORT +else + paster db init -c test-core.ini + paster datastore set-permissions -c test-core.ini | sudo -u postgres psql -p $PG_PORT +fi cd - echo "Installing ckanext-xloader and its requirements..." -python setup.py develop +pip install -r pip-requirements.txt pip install -r requirements.txt pip install -r dev-requirements.txt -echo "Moving test.ini into a subdir..." +python setup.py develop + +echo "Moving test.ini into a subdir... (because the core ini file is referenced as ../ckan/test-core.ini)" mkdir subdir mv test.ini subdir -echo "travis-build.bash is done." +echo "travis-build.bash is done." \ No newline at end of file diff --git a/bin/travis-run.bash b/bin/travis-run.bash index 7a4e5aca..f3317048 100644 --- a/bin/travis-run.bash +++ b/bin/travis-run.bash @@ -1,11 +1,5 @@ -#!/bin/sh -e +#!/bin/bash set -ex -flake8 --version -# stop the build if there are Python syntax errors or undefined names -flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan,ckanext-xloader - -pytest --ckan-ini subdir/test.ini --cov=ckanext.xloader ckanext/xloader/tests - -# strict linting -flake8 . --count --max-complexity=27 --max-line-length=127 --statistics --exclude ckan,ckanext-xloader +pip install future +pytest --ckan-ini=subdir/test.ini --cov=ckanext.xloader --disable-warnings ckanext/xloader/tests From 9b4b3ac99022a208a8f8bf6dbd2bab9738622c1e Mon Sep 17 00:00:00 2001 From: Brett <54408245+kowh-ai@users.noreply.github.com> Date: Tue, 13 Oct 2020 12:54:21 +0200 Subject: [PATCH 2/6] trigger build --- bin/travis-build.bash | 2 +- bin/travis-run.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/travis-build.bash b/bin/travis-build.bash index fc66cf6e..08a23c47 100644 --- a/bin/travis-build.bash +++ b/bin/travis-build.bash @@ -99,4 +99,4 @@ echo "Moving test.ini into a subdir... (because the core ini file is referenced mkdir subdir mv test.ini subdir -echo "travis-build.bash is done." \ No newline at end of file +echo "The travis-build.bash is done." \ No newline at end of file diff --git a/bin/travis-run.bash b/bin/travis-run.bash index f3317048..5636ead6 100644 --- a/bin/travis-run.bash +++ b/bin/travis-run.bash @@ -2,4 +2,4 @@ set -ex pip install future -pytest --ckan-ini=subdir/test.ini --cov=ckanext.xloader --disable-warnings ckanext/xloader/tests +pytest --ckan-ini=subdir/test.ini --cov=ckanext.xloader --disable-warnings ckanext/xloader/tests \ No newline at end of file From 50af95406f565edecbfe62790698bfb3c3dbc35a Mon Sep 17 00:00:00 2001 From: Brett <54408245+kowh-ai@users.noreply.github.com> Date: Wed, 14 Oct 2020 14:50:33 +0200 Subject: [PATCH 3/6] Update alignment --- .travis.yml | 100 ++++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/.travis.yml b/.travis.yml index 18c9fc9a..fd0065b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,8 @@ dist: focal language: python services: - - docker - - redis + - docker + - redis install: bash bin/travis-build.bash script: bash bin/travis-run.bash @@ -14,58 +14,58 @@ stages: - Tests - jobs: - include: - - stage: Flake8 - python: 3.6 - env: FLAKE8=True - install: - - pip install flake8==3.5.0 - - pip install pycodestyle==2.3.0 - script: - - flake8 --version - # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - - flake8 . --count --max-line-length=127 --statistics --exclude ckan --exit-zero +jobs: + include: + - stage: Flake8 + python: 3.6 + env: FLAKE8=True + install: + - pip install flake8==3.5.0 + - pip install pycodestyle==2.3.0 + script: + - flake8 --version + # stop the build if there are Python syntax errors or undefined names + - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + - flake8 . --count --max-line-length=127 --statistics --exclude ckan --exit-zero - - stage: Tests - python: "3.6" - env: CKANVERSION=2.9 - services: - - postgresql - - redis - - docker + - stage: Tests + python: "3.6" + env: CKANVERSION=2.9 + services: + - postgresql + - redis + - docker - - python: "2.7" - env: CKANVERSION=2.9 - services: - - postgresql - - redis - - docker + - python: "2.7" + env: CKANVERSION=2.9 + services: + - postgresql + - redis + - docker - - python: "2.7" - env: CKANVERSION=2.8 - addons: - postgresql: '11' - apt: - sources: - - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main' - packages: - - postgresql-11 + - python: "2.7" + env: CKANVERSION=2.8 + addons: + postgresql: '11' + apt: + sources: + - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main' + packages: + - postgresql-11 - - python: "2.7" - env: CKANVERSION=2.7 - addons: - postgresql: '9.6' - apt: - sources: - - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main' - packages: - - postgresql-9.6 + - python: "2.7" + env: CKANVERSION=2.7 + addons: + postgresql: '9.6' + apt: + sources: + - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main' + packages: + - postgresql-9.6 - cache: - directories: - - $HOME/.cache/pip \ No newline at end of file +cache: + directories: + - $HOME/.cache/pip \ No newline at end of file From d5fb2af7b6ce48ee35af5acbe2809ef20afec503 Mon Sep 17 00:00:00 2001 From: Brett <54408245+kowh-ai@users.noreply.github.com> Date: Wed, 14 Oct 2020 15:02:04 +0200 Subject: [PATCH 4/6] Create pip-requirements.txt --- pip-requirements.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 pip-requirements.txt diff --git a/pip-requirements.txt b/pip-requirements.txt new file mode 100644 index 00000000..b818231f --- /dev/null +++ b/pip-requirements.txt @@ -0,0 +1,8 @@ +ckantoolkit==0.0.3 +pika>=1.1.0 +pyOpenSSL==18.0.0 +redis +requests>=2.11.1 +six>=1.12.0 +messytables==0.15.2 +Unidecode==1.0.22 \ No newline at end of file From 8cfbdaef3a5ab9f8d66d35bfa62a9e5dd455d1d3 Mon Sep 17 00:00:00 2001 From: Brett <54408245+kowh-ai@users.noreply.github.com> Date: Wed, 14 Oct 2020 16:40:49 +0200 Subject: [PATCH 5/6] Update loader.py take out the .encode() for py3 --- ckanext/xloader/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index e70730a6..78a0cb33 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -227,7 +227,7 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): # e is a str but with foreign chars e.g. # 'extra data: "paul,pa\xc3\xbcl"\n' # but logging and exceptions need a normal (7 bit) str - error_str = str(e).decode('ascii', 'replace').encode('ascii', 'replace') + error_str = str(e).decode('ascii', 'replace') logger.warning(error_str) raise LoaderError('Error during the load into PostgreSQL:' ' {}'.format(error_str)) From aa335ed8692d0575dfebb4257554abe4171070b3 Mon Sep 17 00:00:00 2001 From: Brett <54408245+kowh-ai@users.noreply.github.com> Date: Wed, 14 Oct 2020 17:10:42 +0200 Subject: [PATCH 6/6] another update to loader.py take out the .decode() --- ckanext/xloader/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index 78a0cb33..87b6a3e2 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -227,7 +227,7 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): # e is a str but with foreign chars e.g. # 'extra data: "paul,pa\xc3\xbcl"\n' # but logging and exceptions need a normal (7 bit) str - error_str = str(e).decode('ascii', 'replace') + error_str = str(e) logger.warning(error_str) raise LoaderError('Error during the load into PostgreSQL:' ' {}'.format(error_str))