🐘 Postgres

.circleci/config.yml

@@ -9,6 +9,10 @@ jobs:
       # specify the version you desire here
       # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers`
       - image: circleci/python:3.6.1
+      - image: postgres:9.5
+        environment:
+        - POSTGRES_USER=postgres
+        - POSTGRES_DB=test
 
     working_directory: ~/repo
 

Dockerfile

@@ -5,10 +5,8 @@ WORKDIR     /app
 RUN         apt-get update & apt-get install gcc -y
 RUN         pip install -r /app/requirements.txt
 ADD         . /app
+RUN         python /app/setup.py install
 EXPOSE      80
 ENV         FLASK_APP "manage.py"
 ENV         FLASK_CONFIG "production"
-RUN         ["flask", "db", "init"]
-RUN         ["flask", "db", "migrate"]
-RUN         ["flask", "db", "upgrade"]
-CMD         ["gunicorn", "-b", ":80", "--access-logfile", "-",  "manage:app", "--threads", "4"]
+CMD         ["./bin/run.sh"]

Jenkinsfile

@@ -5,41 +5,107 @@ properties([
 
 pipeline {
   agent { label 'docker-slave' }
-stages{
-      stage('Get Code') {
-        steps {
-            checkout scm
-        }
+  stages{
+    stage('Get Code') {
+      steps {
+          deleteDir()
+          checkout scm
       }
-     stage('Build') {
-       steps {
-          sh '''
-          passwd=`aws ecr get-login --region us-east-1 | awk '{ print \$6 }'`
-          docker login -u AWS -p \$passwd 538745987955.dkr.ecr.us-east-1.amazonaws.com/kf-api-dataservice:latest
-          docker build -t 538745987955.dkr.ecr.us-east-1.amazonaws.com/kf-api-dataservice:latest .
-          '''
-       }
+    }
+    stage('Test') {
+     steps {
+       slackSend (color: '#ddaa00', message: ":construction_worker: TESTING STARTED: (${env.BUILD_URL})")
+       sh '''
+       PATH=$WORKSPACE/venv/bin:/usr/local/bin:$PATH
+       virtualenv -p python3 venv
+       . venv/bin/activate
+       which python
+       which pip
+       python -m pip install -r dev-requirements.txt
+       python -m pip install -r requirements.txt
+       python -m pip install -e .
+       export FLASK_APP=manage
+       python -m flask test
+       '''
+       slackSend (color: '#41aa58', message: ":white_check_mark: TESTING COMPLETED: (${env.BUILD_URL})")
      }
-     stage('Publish') {
-       steps {
-          sh '''
-          docker push 538745987955.dkr.ecr.us-east-1.amazonaws.com/kf-api-service:latest
-          '''
+    }
+    stage('Build') {
+      steps {
+        sh '''
+        passwd=`aws ecr get-login --region us-east-1 | awk '{ print \$6 }'`
+        docker login -u AWS -p \$passwd 538745987955.dkr.ecr.us-east-1.amazonaws.com/kf-api-dataservice:latest
+        docker build -t 538745987955.dkr.ecr.us-east-1.amazonaws.com/kf-api-dataservice:latest .
+        '''
+      }
+    }
+    stage('Publish') {
+      steps {
+        sh '''
+        docker push 538745987955.dkr.ecr.us-east-1.amazonaws.com/kf-api-dataservice:latest
+        '''
+        slackSend (color: '#41aa58', message: ":arrow_up: PUSHED IMAGE: (${env.BUILD_URL})")
+      }
+    }
+    stage('Deploy Dev') {
+      when {
+        expression {
+          return env.BRANCH_NAME != 'master';
+        }
+      }
+      steps {
+        slackSend (color: '#005e99', message: ":deploying_dev: DEPLOYING TO DEVELOPMENT: (${env.BUILD_URL})")
+        sh '''
+        rm -rf aws-ecs-service-*
+        git clone git@github.com:kids-first/aws-ecs-service-type-1.git
+        cd aws-ecs-service-type-1/
+        echo "Setting up backend"
+        echo 'key        = "dev/kf-dev-pi-dataservice-us-east-1-RSF"' >> dev.conf
+        terraform init -backend=true -backend-config=dev.conf
+        terraform validate -var 'image=538745987955.dkr.ecr.us-east-1.amazonaws.com/kf-api-dataservice:latest' \
+         -var 'pg_host="kf-dataservice-api-dev.c3siovbugjym.us-east-1.rds.amazonaws.com"' \
+         -var 'db_secret_path="secret/aws/dataservice-api-postgres"' -var 'pg_db_name="kfpostgresdev"' \
+         -var 'task_role_arn="arn:aws:iam::538745987955:role/kfDataserviceApiRole"' -var 'application=dataservice-api' \
+         -var 'service_name="kf-api-dataservice"' -var 'owner="jenkins"' -var-file=dev.tfvar \
+         -var 'vault_role="kf_dataservice_api_role"'
+        terraform apply --auto-approve -var 'image=538745987955.dkr.ecr.us-east-1.amazonaws.com/kf-api-dataservice:latest' \
+         -var 'pg_host="kf-dataservice-api-dev.c3siovbugjym.us-east-1.rds.amazonaws.com"' \
+         -var 'db_secret_path="secret/aws/dataservice-api-postgres"' -var 'pg_db_name="kfpostgresdev"' \
+         -var 'task_role_arn="arn:aws:iam::538745987955:role/kfDataserviceApiRole"' -var 'application=dataservice-api' \
+         -var 'service_name="kf-api-dataservice"' -var 'owner="jenkins"' -var-file=dev.tfvar \
+         -var 'vault_role="kf_dataservice_api_role"'
+        '''
+        slackSend (color: '#41aa58', message: ":white_check_mark: DEPLOYED TO DEVELOPMENT: (${env.BUILD_URL})")
+      }
+    }
+    stage('Deploy QA') {
+      when {
+       expression {
+           return env.BRANCH_NAME == 'master';
        }
      }
-     stage('Deploy Dev') {
-        steps {
-           sh '''
-           rm -rf aws-ecs-service-*
-           git clone git@github.com:kids-first/aws-ecs-service-type-1.git
-           cd aws-ecs-service-type-1/
-           echo "Setting up backend"
-           echo 'key        = "dev/kf-dev-api-dataservice-us-east-1-RSF"' >> dev.conf
-           terraform init -backend=true -backend-config=dev.conf
-           terraform validate -var 'application=api-dataservice' -var 'service_name="kf-api-dataservice"' -var 'owner="jenkins"' -var-file=dev.tfvar
-           terraform apply --auto-approve -var 'application=api-dataservice' -var 'service_name="kf-api-dataservice"' -var 'owner="jenkins"' -var-file=dev.tfvar
-           '''
-        }
+     steps {
+       slackSend (color: '#005e99', message: ":deploying_qa: DEPLOYING TO QA: (${env.BUILD_URL})")
+       sh '''
+       rm -rf aws-ecs-service-*
+       git clone git@github.com:kids-first/aws-ecs-service-type-1.git
+       cd aws-ecs-service-type-1/
+       echo "Setting up backend"
+       echo 'key        = "dev/kf-dev-api-dataservice-us-east-1-RSF"' >> dev.conf
+       terraform init -backend=true -backend-config=dev.conf
+       terraform validate-var 'image=538745987955.dkr.ecr.us-east-1.amazonaws.com/kf-api-dataservice:latest' \
+        -var 'pg_host="kf-dataservice-api-dev.c3siovbugjym.us-east-1.rds.amazonaws.com"' \
+        -var 'db_secret_path="secret/aws/dataservice-api-postgres"' -var 'pg_db_name="kfpostgresdev"' \
+        -var 'task_role_arn="arn:aws:iam::538745987955:role/kfDataserviceApiRole"' -var 'application=dataservice-api' \
+        -var 'service_name="kf-api-dataservice"' -var 'owner="jenkins"' -var-file=dev.tfvar
+       terraform apply --auto-approve -var 'image=538745987955.dkr.ecr.us-east-1.amazonaws.com/kf-api-dataservice:latest' \
+        -var 'pg_host="kf-dataservice-api-dev.c3siovbugjym.us-east-1.rds.amazonaws.com"' \
+        -var 'db_secret_path="secret/aws/dataservice-api-postgres"' -var 'pg_db_name="kfpostgresdev"' \
+        -var 'task_role_arn="arn:aws:iam::538745987955:role/kfDataserviceApiRole"' -var 'application=dataservice-api' \
+        -var 'service_name="kf-api-dataservice"' -var 'owner="jenkins"' -var-file=dev.tfvar
+       '''
+       slackSend (color: '#41aa58', message: ":white_check_mark: DEPLOYED TO QA: (${env.BUILD_URL})")
      }
-   }
+    }
+  }
 }

README.md

@@ -8,7 +8,27 @@ The Kids First Data Service provides a REST API to the Kids First data.
 
 ## Development
 
-Start a development service of your own:
+### Developing against the dataservice
+
+If you're developing an application against the dataservice api and the data
+it contains, the fastest way to get a development service of your own running
+is with docker compose:
+
+```
+git clone git@github.com:kids-first/kf-api-dataservice.git
+cd kf-api-dataservice
+docker-compose up
+```
+
+This will start the dataservice api on port `5000` with a backing postgres
+database initialized with the current datamodel.
+
+### Developing the dataservice api and model
+
+If you're developing features of the api and the data model behind the
+dataservice, you may want finer control over the environment. The following
+are the basics to get you started with a local development environment of
+your own:
 
 ```
 # Get source from github
@@ -18,16 +38,36 @@ cd kf-api-dataservice
 virtualenv venv &&  source venv/bin/activate
 pip install -r dev-requirements.txt
 pip install -r requirements.txt
+pip install -e .
 # Configure the flask application
 export FLASK_APP=manage
-# Setup the database
-flask db init
+# Setup the database (using a dockerized postgres)
+docker run --name dataservice-pg -p 5432:5432 -d postgres:9.5
+docker exec dataservice-pg psql -U postgres -c "CREATE DATABASE dev;"
+flask db migrate 
 flask db upgrade
 # Run the flask web application
 flask run 
 ```
 
-The API should now be available at `localhost:5000/`.
+#### Database
+
+Running postgres inside of a container and binding back to the host should
+be sufficent for most development needs. If you want to access psql
+directly, you can always connect using the following
+(assuming the cointainer is named `dataservice-pg` and the database is `dev`):
+```
+docker exec dataservice-pg psql -U postgres dev
+```
+
+If you'd like to use system install of postgres, or a database running remotely,
+the dataservice can be configured with the following environment variables:
+
+- `PG_HOST` - the host postgres is running on
+- `PG_PORT` - the port postgres is listening on
+- `PG_NAME` - the name of the database in postgres
+- `PG_USER` - the postgres user to connect with
+- `PG_PASS` - the password of the user
 
 ## Documentation
 
@@ -36,7 +76,15 @@ The swagger docs are located at the root `localhost:5000/`.
 ### Generate a Data Model Diagram
 
 An ERD (entity relation diagram) may be found in the `docs/` directory, or may
-be produced for changes to the data schema. To do so requires that 
+be produced for changes to the data schema. To do so requires that the latest
+development version of
+[eralchemy](github.com/Alexis-benoist/eralchemy) be installed:
+
+```
+pip install -e git+git@github.com:Alexis-benoist/eralchemy.git#egg=eralchemy
+```
+
+This also requires
 [GraphViz](https://www.graphviz.org/) be installed as well as
 [PyGraphViz](https://pygraphviz.github.io/). PyGraphViz may have trouble finding
 GraphViz, in which case, see
@@ -58,6 +106,9 @@ Unit tests and pep8 linting is run via `flask test`
 ```
 # Install test dependencies
 pip install -r dev-requirements.txt
+# Setup test database
+docker run --name dataservice-pg -p 5432:5432 -d postgres
+docker exec dataservice-pg psql -U postgres -c "CREATE DATABASE test;"
 # Run tests
 flask test
 ```

bin/run.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+source venv/bin/activate
+flask db upgrade
+exec gunicorn -b :80 --access-logfile - manage:app --threads 4

config.py

@@ -9,6 +9,14 @@ class Config:
     RESTPLUS_MASK_SWAGGER = False
     SQLALCHEMY_TRACK_MODIFICATIONS = False
 
+    PG_HOST = os.environ.get('PG_HOST', 'localhost')
+    PG_PORT = os.environ.get('PG_PORT', 5432)
+    PG_NAME = os.environ.get('PG_NAME', 'dev')
+    PG_USER = os.environ.get('PG_USER', 'postgres')
+    PG_PASS = os.environ.get('PG_PASS', '')
+    SQLALCHEMY_DATABASE_URI = 'postgres://{}:{}@{}:{}/{}'.format(
+                        PG_USER, PG_PASS, PG_HOST, PG_PORT, PG_NAME)
+
     @staticmethod
     def init_app(app):
         pass
@@ -17,34 +25,43 @@ def init_app(app):
 class DevelopmentConfig(Config):
     DEBUG = True
     SSL_DISABLE = True
-    SQLALCHEMY_DATABASE_URI = os.environ.get("DEV_DATABASE_URL") or \
-            "sqlite:///" + os.path.join(basedir, "data-dev.sqlite")
     SQLALCHEMY_TRACK_MODIFICATIONS = True
 
 
 class TestingConfig(Config):
     SERVER_NAME = "localhost"
     TESTING = True
     WTF_CSRF_ENABLED = False
-    SQLALCHEMY_DATABASE_URI = os.environ.get("TEST_DATABASE_URL") or \
-            "sqlite:///" + os.path.join(basedir, "data-test.sqlite")
+    SQLALCHEMY_DATABASE_URI = 'postgres://postgres@localhost:5432/test'
     SQLALCHEMY_TRACK_MODIFICATIONS = True
 
 
 class ProductionConfig(Config):
-    # Should use postgres
-    SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL") or \
-            "sqlite:///" + os.path.join(basedir, "data.sqlite")
-
-    @classmethod
-    def init_app(cls, app):
-        Config.init_app(app)
-
-        # email errors to the administrators
-        import logging
-        from logging.handlers import SMTPHandler
-        credentials = None
-        secure = None
+    @staticmethod
+    def init_app(app):
+        import hvac
+
+        vault_url = os.environ.get('VAULT_URL', 'https://vault:8200/')
+        # Role to authenticate with
+        vault_role = os.environ.get('VAULT_ROLE', 'PostgresRole')
+        # Path for the postgres secret in vault
+        pg_secret = os.environ.get('DB_SECRET', 'secret/postgres')
+        # Retrieve postgres secrets
+        client = hvac.Client(url=vault_url)
+        client.auth_iam(vault_role)
+        secrets = client.read(pg_secret)
+        client.logout()
+
+        pg_user = secrets['data']['user']
+        pg_pass = secrets['data']['password']
+        connection_str = 'postgres://{}:{}@{}:{}/{}'.format(
+                            pg_user,
+                            pg_pass,
+                            Config.PG_HOST,
+                            Config.PG_PORT,
+                            Config.PG_NAME)
+
+        app.config['SQLALCHEMY_DATABASE_URI'] = connection_str
 
 
 class UnixConfig(ProductionConfig):

dataservice/__init__.py

@@ -7,6 +7,9 @@
 from dataservice.api.participant.models import Participant
 from config import config
 
+from sqlalchemy.exc import IntegrityError
+from werkzeug.exceptions import HTTPException
+
 
 def create_app(config_name):
     """
@@ -15,6 +18,7 @@ def create_app(config_name):
     app = Flask(__name__)
     app.url_map.strict_slashes = False
     app.config.from_object(config[config_name])
+    config[config_name].init_app(app)
 
     # Register Flask extensions
     register_extensions(app)
@@ -74,10 +78,12 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
 def register_error_handlers(app):
     """
     Register error handlers
+
+    NB: Exceptions to be handled must be imported in the head of this module
     """
     from dataservice.api import errors
-    from werkzeug.exceptions import HTTPException
     app.register_error_handler(HTTPException, errors.http_error)
+    app.register_error_handler(IntegrityError, errors.integrity_error)
     app.register_error_handler(404, errors.http_error)
     app.register_error_handler(400, errors.http_error)