diff --git a/.gitignore b/.gitignore index 7bbc71c09..a206fd182 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.sqlite +*.swp # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -99,3 +101,6 @@ ENV/ # mypy .mypy_cache/ + +# PyCharm +.idea/ diff --git a/config.py b/config.py new file mode 100644 index 000000000..152adf391 --- /dev/null +++ b/config.py @@ -0,0 +1,69 @@ +import os +basedir = os.path.abspath(os.path.dirname(__file__)) + + +class Config: + HOST = "0.0.0.0" + SSL_DISABLE = os.environ.get("SSL_DISABLE", False) + + RESTPLUS_MASK_SWAGGER = False + + @staticmethod + def init_app(app): + pass + + +class DevelopmentConfig(Config): + DEBUG = True + SSL_DISABLE = True + SQLALCHEMY_DATABASE_URI = os.environ.get("DEV_DATABASE_URL") or \ + "sqlite:///" + os.path.join(basedir, "data-dev.sqlite") + SQLALCHEMY_TRACK_MODIFICATIONS = True + + +class TestingConfig(Config): + SERVER_NAME = "localhost" + TESTING = True + WTF_CSRF_ENABLED = False + SQLALCHEMY_DATABASE_URI = os.environ.get("TEST_DATABASE_URL") or \ + "sqlite:///" + os.path.join(basedir, "data-test.sqlite") + SQLALCHEMY_TRACK_MODIFICATIONS = True + + +class ProductionConfig(Config): + # Should use postgres + SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL") or \ + "sqlite:///" + os.path.join(basedir, "data.sqlite") + + @classmethod + def init_app(cls, app): + Config.init_app(app) + + # email errors to the administrators + import logging + from logging.handlers import SMTPHandler + credentials = None + secure = None + + +class UnixConfig(ProductionConfig): + @classmethod + def init_app(cls, app): + ProductionConfig.init_app(app) + + # log to syslog + import logging + from logging.handlers import SysLogHandler + syslog_handler = SysLogHandler() + syslog_handler.setLevel(logging.WARNING) + app.logger.addHandler(syslog_handler) + + +config = { + "development": DevelopmentConfig, + "testing": TestingConfig, + "production": ProductionConfig, + "unix": UnixConfig, + + "default": DevelopmentConfig +} diff --git a/dataservice/__init__.py b/dataservice/__init__.py new file mode 100644 index 000000000..248005f0c --- /dev/null +++ b/dataservice/__init__.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +"""The app module, containing the app factory function.""" +from flask import Flask + +from dataservice import commands +from dataservice.extensions import db, migrate +from dataservice.api.person.models import Person +from config import config + + +def create_app(config_name): + """ + An application factory + """ + app = Flask(__name__) + app.url_map.strict_slashes = False + app.config.from_object(config[config_name]) + + # Register Flask extensions + register_extensions(app) + register_shellcontext(app) + register_commands(app) + register_blueprints(app) + + return app + + +def register_shellcontext(app): + """ + Register shell context objects + """ + + def shell_context(): + """Shell context objects.""" + return {'db': db, + 'Person': Person} + + app.shell_context_processor(shell_context) + + +def register_commands(app): + """ + Register Click commands + """ + app.cli.add_command(commands.test) + + +def register_extensions(app): + """ + Register Flask extensions + """ + + # SQLAlchemy + db.init_app(app) + + # Migrate + migrate.init_app(app, db) + + +def register_error_handlers(app): + """ + Register error handlers + """ + pass + + +def register_blueprints(app): + from dataservice.api import api_v1 + app.register_blueprint(api_v1) diff --git a/dataservice/api/README.md b/dataservice/api/README.md new file mode 100644 index 000000000..99733b005 --- /dev/null +++ b/dataservice/api/README.md @@ -0,0 +1,49 @@ +Primary API for interacting with the Kid's First Data Model. + +## Service-Wide Standards and Functionality + +### Field Masking + +Partial data fetching is supported via the `X-Fields` header. +To specifiy that only some fields are to be returned, include a bracketed, +coma-delimited list under the `X-Fields` header: + +`X-Fields: {kf_id, name}` + +Brackets may be nested to filter on nested fields: + +`X-Fields: {kf_id, name, type{format, extension}}` + +An asterisk may be used to specify all fields: + +`X-Fields: *` + +Or all sub-fields: + +`X-Fields: {kf_id, type{*}}` + +Or all root fields, but with only some sub-fields: + +`X-Fields: {*, type{format}}` + + + +### Pagination + +``` +{ + pages: [ + { "doc_id": 30, "value": "Lorem" }, + { "doc_id": 31, "value": "ipsum" }, + { "doc_id": 32, "value": "dolor" }, + ... + { "doc_id": 40, "value": "amet" } + ], + from: 30, + to: 40, + results: 10, + total: 1204, + message: "Success", + status: 200 +} +``` diff --git a/dataservice/api/__init__.py b/dataservice/api/__init__.py new file mode 100644 index 000000000..5818d16d4 --- /dev/null +++ b/dataservice/api/__init__.py @@ -0,0 +1,34 @@ +from flask import Blueprint +from flask_restplus import Api +from dataservice.api.person import person_api + +api_v1 = Blueprint('api', __name__, url_prefix='/v1') + +api = Api(api_v1, + title='Kids First Data Service', + description=open('dataservice/api/README.md').read(), + version='0.1', + default='', + default_label='') + +api.add_namespace(person_api) + + +@api.documentation +def redoc_ui(): + """ Uses ReDoc for swagger documentation """ + docs_page = """ + + + API Docs + + + + + + + + + """.format(api.specs_url) + return docs_page diff --git a/dataservice/api/common/__init__.py b/dataservice/api/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dataservice/api/common/id_service.py b/dataservice/api/common/id_service.py new file mode 100644 index 000000000..d3dbe4878 --- /dev/null +++ b/dataservice/api/common/id_service.py @@ -0,0 +1,5 @@ +import uuid + + +def assign_id(): + return str(uuid.uuid4()) diff --git a/dataservice/api/common/model.py b/dataservice/api/common/model.py new file mode 100644 index 000000000..b8e45e858 --- /dev/null +++ b/dataservice/api/common/model.py @@ -0,0 +1,49 @@ +from datetime import datetime +from sqlalchemy.ext.declarative import declared_attr + +from dataservice.extensions import db +from dataservice.api.common.id_service import assign_id + + +class IDMixin: + """ + Defines base ID columns common on all Kids First tables + """ + _id = db.Column(db.Integer(), primary_key=True) + kf_id = db.Column(db.String(32), unique=True, default=assign_id()) + uuid = db.Column(db.String(32), unique=True, default=assign_id()) + + +class TimestampMixin: + """ + Defines the common timestammp columns on all Kids First tables + """ + created_at = db.Column(db.DateTime(), default=datetime.now()) + modified_at = db.Column(db.DateTime(), default=datetime.now()) + + +class HasFileMixin: + @declared_attr + def file_id(cls): + return db.Column('file_id', db.ForeignKey('file._id')) + + @declared_attr + def files(cls): + return db.relationship("File") + + +class Base(db.Model, IDMixin, TimestampMixin): + """ + Defines base SQlAlchemy model class + """ + pass + + +class File(Base): + """ + Defines a file + """ + __tablename__ = "file" + name = db.Column(db.String(32)) + data_type = db.Column(db.String(32)) + size = db.Column(db.Integer(), default=0) diff --git a/dataservice/api/person/README.md b/dataservice/api/person/README.md new file mode 100644 index 000000000..906c983f9 --- /dev/null +++ b/dataservice/api/person/README.md @@ -0,0 +1,21 @@ +A person is one of the central entities of the Kid's First DCC. + +### Fields + +`kf_id` - the unique identifier assigned by Kid's First used as the primary +reference to this Person + +`source_name` - the identifier used by the original contributor of the data + +`date_created` - the date the person`s record was created in the DCC + +`date_modified` - the last date that the record's fields were modified. +Restricted to fields of the entity itself, not any of it's related entities. + +### Identifiers + +The Kid's First DCC assigns a unique, internal identifier of the form: +`KF-P000000` on creation of a new Person. This identifier is used accross the +Kids First Data Service and Data Resource Portal It is expected that the Person +also have an identifier unique to the study it came from. This field is to be +captured in the `source_name` property of the Person upon creation. diff --git a/dataservice/api/person/__init__.py b/dataservice/api/person/__init__.py new file mode 100644 index 000000000..e6bf36b05 --- /dev/null +++ b/dataservice/api/person/__init__.py @@ -0,0 +1 @@ +from .resources import person_api diff --git a/dataservice/api/person/models.py b/dataservice/api/person/models.py new file mode 100644 index 000000000..a5024779e --- /dev/null +++ b/dataservice/api/person/models.py @@ -0,0 +1,16 @@ +from dataservice.extensions import db +from dataservice.api.common.model import Base + + +class Person(Base): + """ + Person entity. + + :param _id: Unique id assigned by RDBMS + :param kf_id: Unique id given by the Kid's First DCC + :param external_id: Name given to person by contributor + :param created_at: Time of object creation + :param modified_at: Last time of object modification + """ + __tablename__ = "person" + external_id = db.Column(db.String(32)) diff --git a/dataservice/api/person/resources.py b/dataservice/api/person/resources.py new file mode 100644 index 000000000..90cb5ad87 --- /dev/null +++ b/dataservice/api/person/resources.py @@ -0,0 +1,110 @@ +from flask import request +from flask_restplus import Namespace, Resource, fields, abort + +from dataservice.extensions import db +from dataservice.api.person import models + +description = open('dataservice/api/person/README.md').read() + +person_api = Namespace(name='persons', description=description) + +from dataservice.api.person.serializers import (person_model, + response_model) + + +@person_api.route('/') +class PersonList(Resource): + @person_api.marshal_with(response_model) + def get(self): + """ + Get all persons + """ + persons = models.Person.query.all() + return {'status': 200, + 'message': '{} persons'.format(len(persons)), + 'content': {'persons': persons}}, 200 + + @person_api.marshal_with(response_model) + @person_api.doc(responses={201: 'person created', + 400: 'invalid data'}) + @person_api.expect(person_model) + def post(self): + """ + Create a new person + + Creates a new person and assigns a Kids First id + """ + body = request.json + person = models.Person(**body) + db.session.add(person) + db.session.commit() + return {'status': 201, + 'message': 'person created', + 'content': {'persons': [person]}}, 201 + + +@person_api.route('/') +class Person(Resource): + @person_api.marshal_with(response_model) + @person_api.doc(responses={200: 'person found', + 404: 'person not found'}) + def get(self, kf_id): + """ + Get a person by id + Gets a person given a Kids First id + """ + person = models.Person.query.filter_by(kf_id=kf_id).one_or_none() + if not person: + self._not_found(kf_id) + + return {'status': 200, + 'message': 'person found', + 'content': {'persons': [person]}}, 200 + + @person_api.marshal_with(response_model) + @person_api.doc(responses={201: 'person updated', + 400: 'invalid data', + 404: 'person not found'}) + @person_api.expect(person_model) + def put(self, kf_id): + """ + Update an existing person + """ + body = request.json + person = models.Person.query.filter_by(kf_id=kf_id).one_or_none() + if not person: + self._not_found(kf_id) + + person.external_id = body.get('external_id') + db.session.commit() + + return {'status': 201, + 'message': 'person updated', + 'content': {'persons': [person]}}, 201 + + @person_api.marshal_with(response_model) + @person_api.doc(responses={204: 'person deleted', + 404: 'person not found'}) + def delete(self, kf_id): + """ + Delete person by id + + Deletes a person given a Kids First id + """ + person = models.Person.query.filter_by(kf_id=kf_id).one_or_none() + if not person: + self._not_found(kf_id) + + db.session.delete(person) + db.session.commit() + return {'status': 200, + 'message': 'person deleted', + 'content': {'persons': [person]}}, 200 + + def _not_found(self, kf_id): + """ + Temporary helper - will do error handling better later + """ + status = 404 + abort(status, "Person with kf_id '{}' not found".format(kf_id), + status=status, content=None) diff --git a/dataservice/api/person/serializers.py b/dataservice/api/person/serializers.py new file mode 100644 index 000000000..0170c3955 --- /dev/null +++ b/dataservice/api/person/serializers.py @@ -0,0 +1,33 @@ +from datetime import datetime +from flask_restplus import fields + +from dataservice.api.person.resources import person_api + +person_model = person_api.model('Person', { + 'kf_id': fields.String( + example='KF00001', + description='ID assigned by Kids First'), + 'created_at': fields.String( + example=datetime.now().isoformat(), + description='Date Person was registered in with the DCC'), + 'modified_at': fields.String( + example=datetime.now().isoformat(), + description='Date of last update to the Persons data'), + 'external_id': fields.String( + example='SUBJ-3993', + description='Identifier used in the original study data') +}) + +person_list = person_api.model("Persons", { + "persons": fields.List(fields.Nested(person_model)) +}) + +response_model = person_api.model('Response', { + 'content': fields.Nested(person_list), + 'status': fields.Integer( + description='HTTP response status code', + example=200), + 'message': fields.String( + description='Additional information about the response', + example='Success') +}) diff --git a/dataservice/commands.py b/dataservice/commands.py new file mode 100644 index 000000000..fef5feec2 --- /dev/null +++ b/dataservice/commands.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +"""Click commands.""" + +import click + + +@click.command() +def test(): + """ Run the unit tests and pep8 checks """ + from subprocess import call + call(["python", "-m", "pytest", "tests"]) + call(["python", "-m", "pytest", "--pep8", "dataservice"]) diff --git a/dataservice/extensions.py b/dataservice/extensions.py new file mode 100644 index 000000000..eb7f765be --- /dev/null +++ b/dataservice/extensions.py @@ -0,0 +1,7 @@ +from flask_migrate import Migrate +from flask_sqlalchemy import SQLAlchemy, Model + + +db = SQLAlchemy() + +migrate = Migrate() diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 000000000..d4cf08270 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,4 @@ +pytest==3.3.1 +pytest-cache==1.0 +pytest-cov==2.5.1 +pytest-pep8==1.0.6 diff --git a/manage.py b/manage.py new file mode 100755 index 000000000..352857b11 --- /dev/null +++ b/manage.py @@ -0,0 +1,9 @@ +#! /usr/bin/env python + +import os +from dataservice import create_app + +app = create_app(os.getenv('FLASK_CONFIG') or 'default') + +if __name__ == '__main__': + app.run() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..1895509bd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,16 @@ +alembic==0.9.6 +aniso8601==1.3.0 +click==6.6 +Flask==0.12.2 +Flask-Migrate==2.1.1 +Flask-Profile==0.2 +flask-restplus==0.10.1 +Flask-SQLAlchemy==2.3.2 +itsdangerous==0.24 +Jinja2==2.8 +Mako==1.0.4 +MarkupSafe==0.23 +python-editor==1.0.3 +six==1.11.0 +SQLAlchemy==1.1.15 +Werkzeug==0.11.10 diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..5d9652b97 --- /dev/null +++ b/setup.py @@ -0,0 +1,9 @@ +from setuptools import setup, find_packages + +setup( + name="kf-api-dataservice", + version="0.1.0", + description="Data Service API", + license="Apache 2", + packages=find_packages() +) diff --git a/tests/person/test_models.py b/tests/person/test_models.py new file mode 100644 index 000000000..32f77ed33 --- /dev/null +++ b/tests/person/test_models.py @@ -0,0 +1,92 @@ +from datetime import datetime +import uuid + +from dataservice.extensions import db +from dataservice.api.person.models import Person +from tests.utils import FlaskTestCase + + +class ModelTest(FlaskTestCase): + """ + Test database model + """ + def _create_person(self, external_id="Test_Person_0"): + """ + Create person with external id + """ + p = Person(external_id=external_id) + db.session.add(p) + db.session.commit() + + return p + + def _get_person(self, kf_id): + """ + Get person by kids first id + """ + return Person.query.filter_by(kf_id=kf_id).one_or_none() + + def test_create_person(self): + """ + Test creation of person + """ + dt = datetime.now() + + self._create_person('Test_Person_0') + + self.assertEqual(Person.query.count(), 1) + new_person = Person.query.first() + self.assertGreater(dt, new_person.created_at) + self.assertGreater(dt, new_person.modified_at) + self.assertEqual(len(new_person.kf_id), 36) + self.assertIs(type(uuid.UUID(new_person.kf_id)), uuid.UUID) + self.assertEqual(new_person.external_id, "Test_Person_0") + + def test_get_person(self): + """ + Test retrieving a person + """ + person = self._create_person('Test_Person_0') + kf_id = person.kf_id + + person = self._get_person(kf_id) + self.assertEqual(Person.query.count(), 1) + self.assertEqual(person.external_id, "Test_Person_0") + self.assertEqual(person.kf_id, kf_id) + + def test_person_not_found(self): + """ + Test retrieving a person that does not exist + """ + person = self._get_person("non_existent_id") + self.assertEqual(person, None) + + def test_update_person(self): + """ + Test updating a person + """ + person = self._create_person('Test_Person_0') + kf_id = person.kf_id + + person = self._get_person(kf_id) + new_name = "Updated-{}".format(person.external_id) + person.external_id = new_name + db.session.commit() + + person = self._get_person(kf_id) + self.assertEqual(person.external_id, new_name) + self.assertEqual(person.kf_id, kf_id) + + def test_delete_person(self): + """ + Test deleting a person + """ + person = self._create_person('Test_Person_0') + kf_id = person.kf_id + + person = self._get_person(kf_id) + db.session.delete(person) + db.session.commit() + + person = self._get_person(kf_id) + self.assertEqual(person, None) \ No newline at end of file diff --git a/tests/person/test_resources.py b/tests/person/test_resources.py new file mode 100644 index 000000000..135a5ab31 --- /dev/null +++ b/tests/person/test_resources.py @@ -0,0 +1,152 @@ +import json +from flask import url_for + +from dataservice.api.person.models import Person +from tests.utils import FlaskTestCase + +PERSONS_PREFIX = 'api.persons' +PERSON_URL = '{}_{}'.format(PERSONS_PREFIX, 'person') +PERSON_LIST_URL = '{}_{}'.format(PERSONS_PREFIX, 'person_list') + + +class PersonTest(FlaskTestCase): + """ + Test person api + """ + + def test_post_person(self): + """ + Test creating a new person + """ + response = self._make_person(external_id="TEST") + self.assertEqual(response.status_code, 201) + + resp = json.loads(response.data.decode("utf-8")) + self._test_response_content(resp, 201) + + self.assertEqual('person created', resp['message']) + + p = Person.query.first() + person = resp['content']['persons'][0] + self.assertEqual(p.kf_id, person['kf_id']) + self.assertEqual(p.external_id, person['external_id']) + + def test_get_not_found(self): + """ + Test get person that does not exist + """ + kf_id = 'non_existent' + response = self.client.get(url_for(PERSON_URL, kf_id=kf_id), + headers=self._api_headers()) + resp = json.loads(response.data.decode("utf-8")) + self.assertEqual(response.status_code, 404) + self._test_response_content(resp, 404) + message = "Person with kf_id '{}' not found".format(kf_id) + self.assertIn(message, resp['message']) + + def test_get_person(self): + """ + Test retrieving a person by id + """ + resp = self._make_person("TEST") + resp = json.loads(resp.data.decode("utf-8")) + kf_id = resp['content']['persons'][0]['kf_id'] + + response = self.client.get(url_for(PERSON_URL, + kf_id=kf_id), + headers=self._api_headers()) + resp = json.loads(response.data.decode("utf-8")) + self.assertEqual(response.status_code, 200) + self._test_response_content(resp, 200) + + person = resp['content']['persons'][0] + + self.assertEqual(kf_id, person['kf_id']) + + def test_get_all_persons(self): + """ + Test retrieving all persons + """ + self._make_person(external_id="MyTestPerson1") + + response = self.client.get(url_for(PERSON_LIST_URL), + headers=self._api_headers()) + status_code = response.status_code + response = json.loads(response.data.decode("utf-8")) + content = response.get('content') + persons = content.get('persons') + self.assertEqual(status_code, 200) + self.assertIs(type(persons), list) + self.assertEqual(len(persons), 1) + + def test_put_person(self): + """ + Test updating an existing person + """ + response = self._make_person(external_id="TEST") + resp = json.loads(response.data.decode("utf-8")) + person = resp['content']['persons'][0] + kf_id = person.get('kf_id') + external_id = person.get('external_id') + + body = { + 'external_id': 'Updated-{}'.format(external_id) + } + response = self.client.put(url_for(PERSON_URL, + kf_id=kf_id), + headers=self._api_headers(), + data=json.dumps(body)) + self.assertEqual(response.status_code, 201) + + resp = json.loads(response.data.decode("utf-8")) + self._test_response_content(resp, 201) + self.assertEqual('person updated', resp['message']) + + p = Person.query.first() + person = resp['content']['persons'][0] + self.assertEqual(p.kf_id, person['kf_id']) + self.assertEqual(p.external_id, person['external_id']) + + def test_delete_person(self): + """ + Test deleting a person by id + """ + resp = self._make_person("TEST") + resp = json.loads(resp.data.decode("utf-8")) + kf_id = resp['content']['persons'][0]['kf_id'] + + response = self.client.delete(url_for(PERSON_URL, + kf_id=kf_id), + headers=self._api_headers()) + + resp = json.loads(response.data.decode("utf-8")) + self.assertEqual(response.status_code, 200) + + response = self.client.get(url_for(PERSON_URL, + kf_id=kf_id), + headers=self._api_headers()) + + resp = json.loads(response.data.decode("utf-8")) + self.assertEqual(response.status_code, 404) + + def _make_person(self, external_id="TEST-0001"): + """ + Convenience method to create a person with a given source name + """ + body = { + 'external_id': external_id + } + response = self.client.post(url_for(PERSON_LIST_URL), + headers=self._api_headers(), + data=json.dumps(body)) + + return response + + def _test_response_content(self, resp, status_code): + """ + Test that response body has expected fields + """ + self.assertIn('content', resp) + self.assertIn('message', resp) + self.assertIn('status', resp) + self.assertEqual(resp['status'], status_code) \ No newline at end of file diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 000000000..2c9110939 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,28 @@ +import json +import unittest +from dataservice import create_app +from dataservice.extensions import db +from flask import url_for + + +class FlaskTestCase(unittest.TestCase): + + def setUp(self): + self.app = create_app("testing") + self.app_context = self.app.app_context() + self.app_context.push() + db.create_all() + + self.client = self.app.test_client() + + def tearDown(self): + db.session.remove() + db.drop_all() + self.app_context.pop() + + def _api_headers(self): + return { + "Accept": "application/json", + "Content-Type": "application/json" + } +