diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 00000000..034a32ea --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,8 @@ +FROM bats/bats + +RUN apk add curl jq + +COPY tests/ /tests/ + +# bats/bats uses bats as the entrypoint +CMD ["-r", "/tests"] diff --git a/Makefile b/Makefile index b2683f0d..60047288 100644 --- a/Makefile +++ b/Makefile @@ -8,12 +8,15 @@ build: requirements: docker-compose run --rm -T app pip --quiet freeze > requirements-freeze.txt - test: - docker-compose build + docker-compose -f docker-compose.yml -f docker-compose.test.yml -f docker-compose.seed.yml build + docker-compose -f docker-compose.yml -f docker-compose.test.yml -f docker-compose.seed.yml up --abort-on-container-exit test up: docker-compose up +up-with-data: + docker-compose up -f docker-compose.yml -f docker-compose.seed.yml + update-dependencies: docker-compose run --rm app pip install -r requirements.txt diff --git a/README.md b/README.md index 5e752455..c3aa2c85 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,13 @@ the `requirements-freeze.txt` for production dependencies. Very little works bey Build and bring up the containers. $ make up + $ make up-with-data [_Gives development environment basic user, organization, and dataset_] -Create an admin user. You'll be prompted for a password. +You may optionally seed the inventory with a default user, organization, and dataset by running the following command in the folder while the docker-compose is still up and has finished running: - $ docker-compose run --rm app paster --plugin=ckan sysadmin add admin -c /etc/ckan/production.ini + $ docker-compose exec app /opt/inventory-app/seed.sh + +_If the user is already created and you would like to rebuild the organization and dataset, you can specify the API key as a second argument to the execution: `docker-compose exec app /opt/inventory-app/seed.sh long-api-key`_ Open CKAN to verify it's working @@ -32,23 +35,36 @@ Open CKAN to verify it's working ### Docker-compose commands -To enter into the container in interactive mode as root: +To enter into the app container in interactive mode as root, you will need to run the following: - $ docker-compose run app bash + $ docker-compose exec app /bin/bash To run a one off command inside the container: - $ docker-compose run app + $ docker-compose exec app {command} Update dependencies. $ make update-dependencies -Update lock file for dependencies. +Update lock file for dependencies. **Because of a version conflict for +repoze.who, special care should be taken to make sure that repoze.who==1.0.18 is +shipped to production in order to be compatible with ckanext-saml2. After +generating the requirements-freeze.txt, manually review the file to make sure +the versions are correct. See https://github.com/GSA/catalog-app/issues/78 for +more details.** $ make requirements +### Live Editing + +To edit CKAN or extension code live, the attached volume needs to be found and used. + +You can find the volume by running `docker volume ls`, but the default is `inventoryapp_ckan`. You can then run `docker volume inspect inventoryapp_ckan` to get the location details on your local machine. You may need to edit permissions to this folder to edit under your current user. Once this is complete, use your preferred editor to manage the code as needed. + +If you restart the service, the volume stays live. It must be removed manually. If you make edits and want to revert, you can run `docker volume rm -f inventoryapp_ckan`. The docker containers need to be stopped and removed before you can run this command. + ### Tests $ make test diff --git a/docker-compose.seed.yml b/docker-compose.seed.yml new file mode 100644 index 00000000..c28e907b --- /dev/null +++ b/docker-compose.seed.yml @@ -0,0 +1,4 @@ +version: '3' +services: + app: + command: /opt/inventory-app/start.sh seed diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 00000000..40998f3d --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,8 @@ +version: '3' +services: + test: + build: + context: . + dockerfile: Dockerfile.test + depends_on: + - app diff --git a/requirements-freeze.txt b/requirements-freeze.txt index 54b144b0..ab37f86e 100644 --- a/requirements-freeze.txt +++ b/requirements-freeze.txt @@ -50,7 +50,7 @@ pyutilib.component.core==4.5.3 PyYAML==5.1.2 redis==2.10.1 repoze.lru==0.6 -repoze.who==2.0 +repoze.who==1.0.18 repoze.who-friendlyform==1.0.8 requests==2.20.0 rfc3987==1.3.8 diff --git a/requirements.txt b/requirements.txt index eb7b5c5c..621524f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,9 @@ pytz tzlocal~=1.0 Markdown~=2.6.11 repoze.lru==0.6 -repoze.who==2.0 +# CKAN requires 2.0 but ckanext-saml2 requires 1.0.18 +# https://github.com/GSA/catalog-app/issues/78 +repoze.who==1.0.18 repoze.who-friendlyform==1.0.8 simplejson==3.3.1 six==1.7.3 diff --git a/seed.sh b/seed.sh new file mode 100755 index 00000000..3930ec0e --- /dev/null +++ b/seed.sh @@ -0,0 +1,137 @@ +#!/bin/bash + +# Can optionally specify an api key if the user is already created. Add as second argument to command. + +set -o errexit +set -o pipefail +set -o nounset + +# The app takes quite a while to startup (solr initialization and +# migrations), close to a minute. Make sure to give it enough time before +# starting the tests. + +hostname="localhost" +port="5000" +retries=20 +while ! nc -z -w 30 "$hostname" "$port" ; do + if [ "$retries" -le 0 ]; then + return 1 + fi + + retries=$(( $retries - 1 )) + echo 'retrying...' + sleep 5 +done + +#If api_key is passed, utilize; if not, create new user +if [ "${1-}" == "" ]; then + echo creating user admin + #Setup various users, organizations, and datasets + if /usr/lib/ckan/bin/paster --plugin=ckan user add admin password=admin email=fake@fake.com -c /etc/ckan/production.ini > /tmp/user_temp.txt ; then + /usr/lib/ckan/bin/paster --plugin=ckan sysadmin add admin -c /etc/ckan/production.ini + api_key=$(grep -oP "apikey.: u.\K.+" /tmp/user_temp.txt | cut -d "'" -f1) + else + api_key=$(/usr/lib/ckan/bin/paster --plugin=ckan user admin -c /etc/ckan/production.ini | grep -oP "apikey=\K.+ " | cut -d " " -f1) + fi + +else + api_key="$1" +fi + +# Adding organization +curl -X POST \ + http://localhost:5000/api/3/action/organization_create \ + -H "authorization: $api_key" \ + -H "cache-control: no-cache" \ + -d '{"description": "Test organization","title": "Test Organization","approval_status": "approved","state": "active","name": "test-organization"}' + +echo '' + +# Adding dataset(s) via API +curl -X POST \ + http://localhost:5000/api/3/action/package_create \ + -H "authorization: $api_key" \ + -H 'cache-control: no-cache' \ + -H 'content-type: application/json' \ + -d ' +{ + "license_title": "License not specified", + "maintainer": null, + "relationships_as_object": [], + "private": true, + "maintainer_email": null, + "num_tags": 1, + "metadata_created": "2019-12-18T19:01:33.429530", + "metadata_modified": "2019-12-18T19:02:54.841495", + "author": null, + "author_email": null, + "state": "active", + "version": null, + "type": "dataset", + "resources": [ + { + "conformsTo": "", + "cache_last_updated": null, + "describedByType": "", + "labels": { + "accessURL new": "Access URL", + "conformsTo": "Conforms To", + "describedBy": "Described By", + "describedByType": "Described By Type", + "format": "Media Type", + "formatReadable": "Format", + "created": "Created" + }, + "webstore_last_updated": null, + "clear_upload": "", + "state": "active", + "size": null, + "describedBy": "", + "hash": "", + "description": "", + "format": "CSV", + "mimetype_inner": null, + "url_type": null, + "formatReadable": "", + "mimetype": null, + "cache_url": null, + "name": "Test Resource", + "created": "2019-12-18T19:02:54.448285", + "url": "https://www.bia.gov/tribal-leaders-csv", + "upload": "", + "webstore_url": null, + "last_modified": null, + "position": 0, + "resource_type": "file" + } + ], + "num_resources": 1, + "tags": [ + { + "vocabulary_id": null, + "state": "active", + "display_name": "test", + "id": "65c76784-e271-4eb1-9778-a738622a1a3d", + "name": "test" + } + ], + "tag_string": "test", + "groups": [], + "license_id": "notspecified", + "relationships_as_subject": [], + "organization": "test-organization", + "isopen": false, + "url": null, + "notes": "The description of the test dataset", + "owner_org": "test-organization", + "bureau_code": "010:00", + "contact_email": "tester@fake.com", + "contact_name": "Tester", + "modified": "2019-12-18", + "public_access_level": "public", + "publisher": "Department of the Interior", + "unique_id": "doi-123456789", + "title": "Test Dataset 1", + "name": "test-dataset-1", + "program_code": "010:001" +}' diff --git a/start.sh b/start.sh index 384639f4..887cf4a0 100755 --- a/start.sh +++ b/start.sh @@ -40,5 +40,15 @@ fi # Run migrations paster --plugin=ckan db upgrade -c /etc/ckan/production.ini +if [ "${1-}" = "seed" ]; then + # Run seed script in new process + echo running seed script... + nohup /opt/inventory-app/seed.sh &> /tmp/nohup.out& + # nohup some_command &> nohup2.out& +fi + +# Work around https://github.com/GSA/catalog-app/issues/78 +pip install -U repoze.who==2.0 + echo starting ckan... exec paster --plugin=ckan serve /etc/ckan/production.ini diff --git a/tests/test.bats b/tests/test.bats new file mode 100644 index 00000000..3a55918a --- /dev/null +++ b/tests/test.bats @@ -0,0 +1,50 @@ +#!/usr/bin/env bats + +function wait_for () { + # The app takes quite a while to startup (solr initialization and + # migrations), close to a minute. Make sure to give it enough time before + # starting the tests. + + local hostname=$1 + local port=$2 + local retries=10 + while ! nc -z -w 30 "$hostname" "$port" ; do + if [ "$retries" -le 0 ]; then + return 1 + fi + + retries=$(( $retries - 1 )) + sleep 5 + done +} + +function test_login_and_datasets () { + sleep 15 # Validate that the seed file has time to implement + curl --silent --fail 'http://app:5000/login_generic?came_from=/user/logged_in' --compressed -H 'Content-Type: application/x-www-form-urlencoded' -H 'Origin: http://app:5000' -H 'Referer: http://app:5000/user/login' --data 'login=admin&password=admin' --cookie-jar ./cookie-jar + + dataset_success=$(curl --fail --location --request GET 'http://app:5000/api/3/action/package_show?id=test-dataset-1' --cookie ./cookie-jar | grep -o '"success": true') + + if [ "$dataset_success" = '"success": true' ]; then + return 0; + else + return 1; + fi +} + +@test "app container is up" { + wait_for app 5000 +} + +@test "/user/login is up" { + curl --silent --fail http://app:5000/user/login +} + +@test "data is accessible for user" { + test_login_and_datasets +} + +@test "data is inaccessible to public" { + run curl --fail --location --request GET 'http://app:5000/api/3/action/package_show?id=test-dataset-1' + # Validate output is 22, curl response for 403 (Forbidden) + [ "$status" -eq 22 ] +}