Merge pull request #3 from PsychoinformaticsLab/dockerize

Set up docker-compose, and other boilerplate

.env.example

@@ -0,0 +1,3 @@
+APP_SETTINGS=neurostuff.config.DevelopmentConfig
+COMPOSE_CONVERT_WINDOWS_PATHS=1
+POSTGRES_PASSWORD=example

.gitignore

@@ -1,4 +1,5 @@
 historical/
+neurostuff/config.py
 
 #
 # Byte-compiled / optimized / DLL files

.travis.yml

@@ -0,0 +1,21 @@
+matrix:
+  include:
+    - language: python
+      python:
+        - "3.6"
+      install:
+        - pip install -U setuptools==45
+        - pip install -r neurostuff/requirements.txt
+        - pip install pytest-cov
+      script:
+        - python -m pytest neurostuff/ --cov=./
+      env:
+      - APP_SETTINGS=neurostuff.config.app.TravisConfig PYTHONHASHSEED=0
+      addons:
+        postgresql: "9.4"
+      before_script:
+        - cp neurostuff/example_config.py neurostuff/config.py
+        - psql -c 'create database travis_ci_test;' -U postgres
+        - CI="true"
+      after_success:
+        - bash <(curl -s https://codecov.io/bash)

.zenodo.json

@@ -0,0 +1,20 @@
+{
+  "creators": [
+     {
+      "affiliation": "University of Texas at Austin",
+      "name": "Yarkoni, Tal",
+      "orcid": "0000-0002-6558-5113"
+     },
+    {
+      "affiliation": "University of Texas at Austin",
+      "name": "De La Vega, Alejandro",
+      "orcid": "0000-0001-9062-3778"
+    },
+  ],
+  "keywords": [
+    "neuroimaging",
+    "meta-analysis",
+    "fMRI"
+  ],
+  "upload_type": "software"
+}

README.md

@@ -0,0 +1,49 @@
+# neurostuff
+
+[Add badges]
+
+Requirements: Docker and docker-compose.
+
+## Configuration
+First, set up the main environment variables in `.env` (see: `.env.example`).
+
+    cp .env.example .env
+
+Next, set up the Flask server's environment variables:
+
+    cp neurostuff/example_config.py neurostuff/config.py
+
+
+Edit both of these template files to set the correct variables
+
+## Initalizing backend
+Build the containers and start services using the development configuration:
+
+    docker-compose build
+    docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d
+
+The server should now be running at http://localhost/
+
+Next, initialize, migrate and upgrade the database migrations.
+
+    docker-compose exec neurostuff bash
+    rm -rf /migrations/migrations
+    python manage.py db init
+    python manage.py db migrate
+    python manage.py db upgrade
+
+Finally, add an admin user, and ingest data
+
+    python manage.py add_user admin@neurostuff.org password
+    python manage.py ingest_neurosynth
+
+
+## Maintaining docker image and db
+If you make a change to /neurostuff, you should be able to simply restart the server.
+
+    docker-compose restart neurostuff
+
+If you need to upgrade the db after changing any models:
+
+    docker-compose exec neurostuff python manage.py db migrate
+    docker-compose exec neurostuff python manage.py db upgrade

codecov.yml

@@ -0,0 +1,3 @@
+coverage:
+  ignore:
+    - "neurostuff/tests/*"

docker-compose.dev.yml

@@ -0,0 +1,9 @@
+version: "2"
+services:
+  nginx:
+    volumes:
+      - ./nginx/nginx.dev.conf:/etc/nginx/nginx.conf:ro
+
+  neurostuff:
+    command: /usr/local/bin/gunicorn -w 2 -b :8000 neurostuff.core:app --log-level debug --timeout 120 --reload
+    restart: "no"

docker-compose.yml

@@ -0,0 +1,45 @@
+version: "2"
+services:
+  neurostuff:
+    restart: always
+    build: ./neurostuff
+    expose:
+      - "8000"
+    volumes:
+      - ./postgres/migrations:/migrations
+      - ./:/neurostuff
+    command: /usr/local/bin/gunicorn -w 2 -b :8000 neurostuff.core:app --log-level debug --timeout 120
+    env_file:
+      - .env
+
+  nginx:
+    restart: always
+    build: ./nginx
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./nginx/certs:/etc/letsencrypt
+      - ./nginx/certs-data:/data/letsencrypt
+      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./nginx/sites-enabled:/etc/nginx/sites-enabled:ro
+    volumes_from:
+      - neurostuff
+
+  postgres:
+    restart: always
+    build: ./postgres
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    expose:
+      - '5432'
+    environment:
+      - POSTGRES_DB=neurostuff
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    env_file:
+      - .env
+
+volumes:
+  certs:
+  certs-data:
+  postgres_data:

manage.py

@@ -0,0 +1,50 @@
+"""
+    Command line management tools.
+"""
+import os
+
+from flask_script import Manager, Shell
+from flask_migrate import Migrate, MigrateCommand
+from flask_security.utils import encrypt_password
+
+from neurostuff.core import app, db, user_datastore
+from neurostuff import ingest
+from neurostuff import models
+
+app.config.from_object(os.environ['APP_SETTINGS'])
+migrate = Migrate(app, db, directory=app.config['MIGRATIONS_DIR'])
+manager = Manager(app)
+
+
+def _make_context():
+    return dict(app=app, db=db, ms=models)
+
+
+manager.add_command('db', MigrateCommand)
+manager.add_command("shell", Shell(make_context=_make_context))
+
+
+@manager.command
+def add_user(email, password):
+    """ Add a user to the database.
+    email - A valid email address (primary login key)
+    password - Any string
+    """
+    user_datastore.create_user(
+        email=email, password=encrypt_password(password))
+
+    db.session.commit()
+
+
+@manager.command
+def ingest_neurosynth(max_rows=None):
+    ingest.ingest_neurosynth(max_rows=max_rows)
+
+
+@manager.command
+def ingest_neurovault(verbose=False, limit=20):
+    ingest.ingest_neurovault(verbose=verbose, limit=limit)
+
+
+if __name__ == '__main__':
+    manager.run()

neurostuff/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.6-stretch
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+RUN apt-get -qq update
+
+COPY requirements.txt /usr/src/app/
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . /usr/src/app
+
+WORKDIR /neurostuff

neurostuff/core.py

@@ -1,36 +1,31 @@
+import os
 from flask import Flask
 from flask_security import Security, SQLAlchemyUserDatastore
 from flask_dance.consumer.storage.sqla import SQLAlchemyStorage
-from flask_dance.contrib.github import make_github_blueprint, github
+from flask_dance.contrib.github import make_github_blueprint
 from flask_cors import CORS
 
 from .database import init_db
 from .models import User, Role, OAuth
 
 
 app = Flask(__name__)
-
-# enable CORS for development
-CORS(app, expose_headers='X-Total-Count')
-
-# Move this stuff out when it gets big
-app.debug = True
-app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///development.db'
-app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
-app.config['APISPEC_SWAGGER_URL'] = '/api/swagger.json'
-app.config['APISPEC_SWAGGER_UI_URL'] = '/api/'
+app.config.from_object(os.environ['APP_SETTINGS'])
 db = init_db(app)
 
+# Enable CORS
+cors = CORS(app, expose_headers='X-Total-Count')
+
 # Flask-Security
 user_datastore = SQLAlchemyUserDatastore(db, User, Role)
 security = Security(app, user_datastore)
 
 # Flask-Dance (OAuth)
 from . import oauth
-app.secret_key = "temporary"
+app.secret_key = app.config['DANCE_SECRET_KEY']
 blueprint = make_github_blueprint(
-    client_id="d5372fa09c97d5a98a84",
-    client_secret="dee86c2c9344f00a31d83854eb135e94957ac494",
+    client_id=app.config['GITHUB_CLIENT_ID'],
+    client_secret=app.config['GITHUB_CLIENT_SECRET'],
 )
 app.register_blueprint(blueprint, url_prefix="/login")
 blueprint.storage = SQLAlchemyStorage(OAuth, db.session)

neurostuff/example_config.py

@@ -0,0 +1,44 @@
+""" This is an EXAMPLE config file
+ Rename this file to app.py and set variables
+"""
+import os
+
+
+class Config(object):
+    SERVER_NAME = 'localhost'  # Set to external server name in production
+
+    MIGRATIONS_DIR = '/migrations/migrations'
+    SQLALCHEMY_TRACK_MODIFICATIONS = False
+    WTF_CSRF_ENABLED = False
+
+    POSTGRES_PASSWORD = os.environ['POSTGRES_PASSWORD']
+    DB_NAME = 'neurostuff'
+    SQLALCHEMY_DATABASE_URI = "postgres://postgres:" \
+        f"{POSTGRES_PASSWORD}@postgres:5432/{DB_NAME}"
+    PROPAGATE_EXCEPTIONS = True
+
+    GITHUB_CLIENT_ID = "github-id"
+    GITHUB_CLIENT_SECRET = "github-secret"
+    DANCE_SECRET_KEY = "temporary"
+
+
+class ProductionConfig(Config):
+    ENV = 'production'
+
+
+class DevelopmentConfig(Config):
+    ENV = 'development'
+    DEBUG = True
+
+
+class TestingConfig(Config):
+    ENV = 'testing'
+    TESTING = True
+
+
+class DockerTestConfig(TestingConfig):
+    SQLALCHEMY_DATABASE_URI = 'postgres://postgres@postgres:5432/scout_test'
+
+
+class TravisConfig(TestingConfig):
+    SQLALCHEMY_DATABASE_URI = "postgresql://postgres@localhost/travis_ci_test"

scripts/ingest_data.py → neurostuff/ingest/__init__.py

@@ -4,23 +4,14 @@
 import re
 import os.path as op
 from pathlib import Path
-from datetime import datetime
 from dateutil.parser import parse as parse_date
 import tarfile
-import tempfile
 
 import pandas as pd
 import requests
 
-from neurostuff.models import Study, Analysis, Condition, Image, User, Point
-from neurostuff.core import db, user_datastore
-
-
-def reset_database():
-    db.drop_all()
-    db.create_all()
-    user_datastore.create_user(email='admin@neurostuff.org', password='password')
-    db.session.commit()
+from neurostuff.models import Study, Analysis, Image, User, Point
+from neurostuff.core import db
 
 
 def ingest_neurovault(verbose=False, limit=20):
@@ -90,8 +81,8 @@ def ingest_neurosynth(max_rows=None):
     # response = requests.get(url, stream=True)
 
     # with tempfile.TemporaryFile() as tf:
-        # tf.write(response.raw.read())
-        # tf.seek(0)
+    #    tf.write(response.raw.read())
+    #   tf.seek(0)
 
     path = Path(__file__).parent / '..' / 'data' / 'data_0.7.July_2018.tar.gz'
     with open(path, 'rb') as tf:
@@ -109,7 +100,7 @@ def ingest_neurosynth(max_rows=None):
                 'year': int(row['year']),
                 'journal': row['journal']
             }
-            s = Study(name=row['title'], metadata_ = md, doi=doi, user=user)
+            s = Study(name=row['title'], metadata=md, doi=doi, user=user)
             analyses = []
             points = []
             for t_id, df in study_df.groupby('table_id'):
@@ -121,8 +112,3 @@ def ingest_neurosynth(max_rows=None):
                     points.append(point)
             db.session.add_all([s] + analyses + points)
             db.session.commit()
-
-
-# reset_database()
-# ingest_neurovault(limit=20)
-ingest_neurosynth(1000)

neurostuff/lib64

@@ -0,0 +1 @@
+lib