Merge pull request #13 from GSA/develop

Release

Dockerfile.test

@@ -0,0 +1,8 @@
+FROM bats/bats
+
+RUN apk add curl jq
+
+COPY tests/ /tests/
+
+# bats/bats uses bats as the entrypoint
+CMD ["-r", "/tests"]

Makefile

@@ -8,12 +8,15 @@ build:
 requirements:
 	docker-compose run --rm -T app pip --quiet freeze > requirements-freeze.txt
 
-
 test:
-	docker-compose build
+	docker-compose -f docker-compose.yml -f docker-compose.test.yml -f docker-compose.seed.yml build
+	docker-compose -f docker-compose.yml -f docker-compose.test.yml -f docker-compose.seed.yml up --abort-on-container-exit test
 
 up:
 	docker-compose up
 
+up-with-data:
+	docker-compose up -f docker-compose.yml -f docker-compose.seed.yml
+
 update-dependencies:
 	docker-compose run --rm app pip install -r requirements.txt

README.md

@@ -21,34 +21,50 @@ the `requirements-freeze.txt` for production dependencies. Very little works bey
 Build and bring up the containers.
 
     $ make up
+    $ make up-with-data [_Gives development environment basic user, organization, and dataset_]
 
-Create an admin user. You'll be prompted for a password.
+You may optionally seed the inventory with a default user, organization, and dataset by running the following command in the folder while the docker-compose is still up and has finished running:
 
-    $ docker-compose run --rm app paster --plugin=ckan sysadmin add admin -c /etc/ckan/production.ini
+    $ docker-compose exec app /opt/inventory-app/seed.sh
+
+_If the user is already created and you would like to rebuild the organization and dataset, you can specify the API key as a second argument to the execution: `docker-compose exec app /opt/inventory-app/seed.sh long-api-key`_
 
 Open CKAN to verify it's working
 
     $ open http://localhost:5000
 
 ### Docker-compose commands
 
-To enter into the container in interactive mode as root:
+To enter into the app container in interactive mode as root, you will need to run the following:
 
-    $ docker-compose run app bash
+    $ docker-compose exec app /bin/bash
 
 To run a one off command inside the container:
 
-    $ docker-compose run app <command>
+    $ docker-compose exec app {command}
 
 Update dependencies.
 
     $ make update-dependencies
 
-Update lock file for dependencies.
+Update lock file for dependencies. **Because of a version conflict for
+repoze.who, special care should be taken to make sure that repoze.who==1.0.18 is
+shipped to production in order to be compatible with ckanext-saml2. After
+generating the requirements-freeze.txt, manually review the file to make sure
+the versions are correct. See https://github.com/GSA/catalog-app/issues/78 for
+more details.**
 
     $ make requirements
 
 
+### Live Editing
+
+To edit CKAN or extension code live, the attached volume needs to be found and used.
+
+You can find the volume by running `docker volume ls`, but the default is `inventoryapp_ckan`. You can then run `docker volume inspect inventoryapp_ckan` to get the location details on your local machine. You may need to edit permissions to this folder to edit under your current user. Once this is complete, use your preferred editor to manage the code as needed.
+
+If you restart the service, the volume stays live. It must be removed manually. If you make edits and want to revert, you can run `docker volume rm -f inventoryapp_ckan`. The docker containers need to be stopped and removed before you can run this command.
+
 ### Tests
 
     $ make test

docker-compose.seed.yml

@@ -0,0 +1,4 @@
+version: '3'
+services:
+  app:
+    command: /opt/inventory-app/start.sh seed

docker-compose.test.yml

@@ -0,0 +1,8 @@
+version: '3'
+services:
+  test:
+    build:
+      context: .
+      dockerfile: Dockerfile.test
+    depends_on:
+      - app

requirements-freeze.txt

@@ -50,7 +50,7 @@ pyutilib.component.core==4.5.3
 PyYAML==5.1.2
 redis==2.10.1
 repoze.lru==0.6
-repoze.who==2.0
+repoze.who==1.0.18
 repoze.who-friendlyform==1.0.8
 requests==2.20.0
 rfc3987==1.3.8

requirements.txt

@@ -41,7 +41,9 @@ pytz
 tzlocal~=1.0
 Markdown~=2.6.11
 repoze.lru==0.6
-repoze.who==2.0
+# CKAN requires 2.0 but ckanext-saml2 requires 1.0.18
+# https://github.com/GSA/catalog-app/issues/78
+repoze.who==1.0.18
 repoze.who-friendlyform==1.0.8
 simplejson==3.3.1
 six==1.7.3

seed.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+
+# Can optionally specify an api key if the user is already created. Add as second argument to command.
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# The app takes quite a while to startup (solr initialization and
+# migrations), close to a minute. Make sure to give it enough time before
+# starting the tests.
+
+hostname="localhost"
+port="5000"
+retries=20
+while ! nc -z -w 30 "$hostname" "$port" ; do
+  if [ "$retries" -le 0 ]; then
+    return 1
+  fi
+
+  retries=$(( $retries - 1 ))
+  echo 'retrying...'
+  sleep 5
+done
+
+#If api_key is passed, utilize; if not, create new user
+if [ "${1-}" == "" ]; then
+  echo creating user admin
+  #Setup various users, organizations, and datasets
+  if /usr/lib/ckan/bin/paster --plugin=ckan user add admin password=admin email=fake@fake.com -c /etc/ckan/production.ini > /tmp/user_temp.txt ; then
+    /usr/lib/ckan/bin/paster --plugin=ckan sysadmin add admin -c /etc/ckan/production.ini
+    api_key=$(grep -oP "apikey.: u.\K.+" /tmp/user_temp.txt | cut -d "'" -f1)
+  else
+    api_key=$(/usr/lib/ckan/bin/paster --plugin=ckan user admin -c /etc/ckan/production.ini | grep -oP "apikey=\K.+ " | cut -d " " -f1)
+  fi
+
+else
+  api_key="$1"
+fi
+
+# Adding organization
+curl -X POST \
+  http://localhost:5000/api/3/action/organization_create \
+  -H "authorization: $api_key" \
+  -H "cache-control: no-cache" \
+  -d '{"description": "Test organization","title": "Test Organization","approval_status": "approved","state": "active","name": "test-organization"}'
+
+echo ''
+
+# Adding dataset(s) via API
+curl -X POST \
+  http://localhost:5000/api/3/action/package_create \
+  -H "authorization: $api_key" \
+  -H 'cache-control: no-cache' \
+  -H 'content-type: application/json' \
+  -d '
+{
+  "license_title": "License not specified",
+  "maintainer": null,
+  "relationships_as_object": [],
+  "private": true,
+  "maintainer_email": null,
+  "num_tags": 1,
+  "metadata_created": "2019-12-18T19:01:33.429530",
+  "metadata_modified": "2019-12-18T19:02:54.841495",
+  "author": null,
+  "author_email": null,
+  "state": "active",
+  "version": null,
+  "type": "dataset",
+  "resources": [
+    {
+      "conformsTo": "",
+      "cache_last_updated": null,
+      "describedByType": "",
+      "labels": {
+        "accessURL new": "Access URL",
+        "conformsTo": "Conforms To",
+        "describedBy": "Described By",
+        "describedByType": "Described By Type",
+        "format": "Media Type",
+        "formatReadable": "Format",
+        "created": "Created"
+      },
+      "webstore_last_updated": null,
+      "clear_upload": "",
+      "state": "active",
+      "size": null,
+      "describedBy": "",
+      "hash": "",
+      "description": "",
+      "format": "CSV",
+      "mimetype_inner": null,
+      "url_type": null,
+      "formatReadable": "",
+      "mimetype": null,
+      "cache_url": null,
+      "name": "Test Resource",
+      "created": "2019-12-18T19:02:54.448285",
+      "url": "https://www.bia.gov/tribal-leaders-csv",
+      "upload": "",
+      "webstore_url": null,
+      "last_modified": null,
+      "position": 0,
+      "resource_type": "file"
+    }
+  ],
+  "num_resources": 1,
+  "tags": [
+    {
+      "vocabulary_id": null,
+      "state": "active",
+      "display_name": "test",
+      "id": "65c76784-e271-4eb1-9778-a738622a1a3d",
+      "name": "test"
+    }
+  ],
+  "tag_string": "test",
+  "groups": [],
+  "license_id": "notspecified",
+  "relationships_as_subject": [],
+  "organization": "test-organization",
+  "isopen": false,
+  "url": null,
+  "notes": "The description of the test dataset",
+  "owner_org": "test-organization",
+  "bureau_code": "010:00",
+  "contact_email": "tester@fake.com",
+  "contact_name": "Tester",
+  "modified": "2019-12-18",
+  "public_access_level": "public",
+  "publisher": "Department of the Interior",
+  "unique_id": "doi-123456789",
+  "title": "Test Dataset 1",
+  "name": "test-dataset-1",
+  "program_code": "010:001"
+}'

start.sh

@@ -40,5 +40,15 @@ fi
 # Run migrations
 paster --plugin=ckan db upgrade -c /etc/ckan/production.ini
 
+if [ "${1-}" = "seed" ]; then
+  # Run seed script in new process
+  echo running seed script...
+  nohup /opt/inventory-app/seed.sh &> /tmp/nohup.out&
+  # nohup some_command &> nohup2.out&
+fi
+
+# Work around https://github.com/GSA/catalog-app/issues/78
+pip install -U repoze.who==2.0
+
 echo starting ckan...
 exec paster --plugin=ckan serve /etc/ckan/production.ini

tests/test.bats

@@ -0,0 +1,50 @@
+#!/usr/bin/env bats
+
+function wait_for () {
+  # The app takes quite a while to startup (solr initialization and
+  # migrations), close to a minute. Make sure to give it enough time before
+  # starting the tests.
+
+  local hostname=$1
+  local port=$2
+  local retries=10
+  while ! nc -z -w 30 "$hostname" "$port" ; do
+    if [ "$retries" -le 0 ]; then
+      return 1
+    fi
+
+    retries=$(( $retries - 1 ))
+    sleep 5
+  done
+}
+
+function test_login_and_datasets () {
+  sleep 15 # Validate that the seed file has time to implement
+  curl --silent --fail 'http://app:5000/login_generic?came_from=/user/logged_in' --compressed -H 'Content-Type: application/x-www-form-urlencoded' -H 'Origin: http://app:5000' -H 'Referer: http://app:5000/user/login' --data 'login=admin&password=admin' --cookie-jar ./cookie-jar
+
+  dataset_success=$(curl --fail --location --request GET 'http://app:5000/api/3/action/package_show?id=test-dataset-1' --cookie ./cookie-jar | grep -o '"success": true')
+
+  if [ "$dataset_success" = '"success": true' ]; then
+    return 0;
+  else
+    return 1;
+  fi
+}
+
+@test "app container is up" {
+  wait_for app 5000
+}
+
+@test "/user/login is up" {
+  curl --silent --fail http://app:5000/user/login
+}
+
+@test "data is accessible for user" {
+  test_login_and_datasets
+}
+
+@test "data is inaccessible to public" {
+  run curl --fail --location --request GET 'http://app:5000/api/3/action/package_show?id=test-dataset-1'
+  # Validate output is 22, curl response for 403 (Forbidden)
+  [ "$status" -eq 22 ]
+}