From ea389e3dff5a1aac3b49f9396b893a596c44477a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Greinhofer?= Date: Fri, 9 Nov 2018 19:11:47 -0600 Subject: [PATCH] Replace DRF with connexion `connexion` is a framework that automagically handles HTTP requests based on OpenAPI Specification. The routes and the documentation are autogenerated. Models are being validated against the specification. * Remove all django files * Update requirements * Create function to generate and configure a connexion app * Implement a way to manage the settings similar to what was done for DRF * Fix the unit tests * Fix integration tests * Update the docker image * Fix the wheel package * Update and clean up Makefile * Update the GitHub PR template Fixes request-yo-racks/api#60 --- .coveragerc | 11 +- .github/PULL_REQUEST_TEMPLATE.md | 7 +- Dockerfile | 5 +- Makefile | 30 +--- api/apps/api/models.py | 1 - api/apps/api/serializers.py | 1 - api/apps/api/urls.py | 14 -- api/apps/api/views.py | 63 ------- api/assets/css/styles.css | 1 - api/assets/js/scripts.js | 1 - api/celery/tasks.py | 9 +- api/{apps => collectors}/__init__.py | 0 api/{apps/api => }/collectors/base.py | 2 +- api/{apps/api => }/collectors/generic.py | 6 +- api/{apps/api => }/collectors/google.py | 8 +- api/{apps/api => }/collectors/yelp.py | 8 +- api/connexion_server.py | 12 ++ api/connexion_utils.py | 101 +++++++++++ api/{apps/api => controller}/__init__.py | 0 api/controller/health.py | 8 + api/controller/place.py | 16 ++ api/controller/places.py | 20 +++ api/settings/__init__.py | 1 - api/settings/base.py | 164 ------------------ api/settings/common.py | 14 ++ api/settings/local.py | 23 +-- api/settings/production.py | 18 +- api/settings/testing.py | 20 --- api/static.py | 8 - api/templates/404.html | 7 - api/templates/500.html | 7 - api/templates/base.html | 42 ----- api/urls.py | 12 -- api/wsgi.py | 20 +-- charts/values.minikube.yaml | 2 +- docker/docker-entrypoint.sh | 6 - docs/source/collectors/collectors.rst | 8 +- openapi/openapi.yaml | 2 +- requirements-dev.txt | 12 +- requirements.txt | 16 +- setup.cfg | 4 + .../apps/api/collectors => tests}/__init__.py | 0 .../api/tests => tests/celery}/__init__.py | 0 .../tests => tests/celery}/test_tasks.py | 14 +- .../tests => tests}/collectors/test_base.py | 10 +- .../collectors/test_generic.py | 6 +- .../tests => tests}/collectors/test_google.py | 8 +- .../tests => tests}/collectors/test_yelp.py | 8 +- ...google_collector_details_epoch_coffee.json | 0 .../google_collector_search_epoch_coffee.json | 0 .../test_integration_collectors.py | 12 +- tox.ini | 2 +- 52 files changed, 263 insertions(+), 507 deletions(-) delete mode 100644 api/apps/api/models.py delete mode 100644 api/apps/api/serializers.py delete mode 100644 api/apps/api/urls.py delete mode 100644 api/apps/api/views.py delete mode 100644 api/assets/css/styles.css delete mode 100644 api/assets/js/scripts.js rename api/{apps => collectors}/__init__.py (100%) rename api/{apps/api => }/collectors/base.py (99%) rename api/{apps/api => }/collectors/generic.py (96%) rename api/{apps/api => }/collectors/google.py (93%) rename api/{apps/api => }/collectors/yelp.py (94%) create mode 100644 api/connexion_server.py create mode 100644 api/connexion_utils.py rename api/{apps/api => controller}/__init__.py (100%) create mode 100644 api/controller/health.py create mode 100644 api/controller/place.py create mode 100644 api/controller/places.py delete mode 100644 api/settings/__init__.py delete mode 100644 api/settings/base.py create mode 100644 api/settings/common.py delete mode 100644 api/settings/testing.py delete mode 100644 api/static.py delete mode 100644 api/templates/404.html delete mode 100644 api/templates/500.html delete mode 100644 api/templates/base.html delete mode 100644 api/urls.py rename {api/apps/api/collectors => tests}/__init__.py (100%) rename {api/apps/api/tests => tests/celery}/__init__.py (100%) rename {api/celery/tests => tests/celery}/test_tasks.py (92%) rename {api/apps/api/tests => tests}/collectors/test_base.py (83%) rename {api/apps/api/tests => tests}/collectors/test_generic.py (95%) rename {api/apps/api/tests => tests}/collectors/test_google.py (99%) rename {api/apps/api/tests => tests}/collectors/test_yelp.py (97%) rename {api/apps/api/tests => tests}/google_collector_details_epoch_coffee.json (100%) rename {api/apps/api/tests => tests}/google_collector_search_epoch_coffee.json (100%) rename {api/apps/api/tests => tests}/test_integration_collectors.py (94%) diff --git a/.coveragerc b/.coveragerc index ed8e707..6fdd0c5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,19 +1,12 @@ [run] omit = *__init__.py - api/apps/api/models.py - api/apps/api/serializers.py - api/apps/api/tests/* - api/apps/api/urls.py - api/apps/api/views.py api/celery/celery.py api/celery/celery_settings.py - api/celery/tests/* + api/connexion_* + api/controller/* api/settings/* - api/static.py - api/urls.py api/wsgi.py - [report] fail_under = 90.0 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1a628fd..a2312be 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -20,14 +20,13 @@ Add a screenshot if applicable. --> + +Why is this change required? What problem does it solve? --> -How Has This Been Tested? -------------------------- - - - - - {% block css %} - - {% endblock %} - - - - -
- - {% block content %} -

Use this document as a way to quick start any new project.

- {% endblock content %} - -
- - - - - {% block javascript %} - - {% endblock javascript %} - - diff --git a/api/urls.py b/api/urls.py deleted file mode 100644 index a856e41..0000000 --- a/api/urls.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Define the main URLs.""" -from django.conf.urls import url -from django.conf.urls import include -from django.contrib import admin - -from api.apps.api import views - -urlpatterns = [ - url(r'^admin/', admin.site.urls), - url(r'^$', views.Health.as_view()), - url(r'^', include('api.apps.api.urls')), -] diff --git a/api/wsgi.py b/api/wsgi.py index ec1bb8a..b08386f 100644 --- a/api/wsgi.py +++ b/api/wsgi.py @@ -1,18 +1,6 @@ -""" -WSGI config for api project. +"""Define the WSGI app.""" -It exposes the WSGI callable as a module-level variable named ``application``. -""" -import os -from os.path import abspath, dirname -from sys import path +from api.connexion_utils import create_connexion_app -from django.core.wsgi import get_wsgi_application -from whitenoise.django import DjangoWhiteNoise - -SITE_ROOT = dirname(dirname(abspath(__file__))) -path.append(SITE_ROOT) - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "api.settings.production") -application = get_wsgi_application() -application = DjangoWhiteNoise(application) +app = create_connexion_app() +application = app.app diff --git a/charts/values.minikube.yaml b/charts/values.minikube.yaml index c0754ae..56f87aa 100644 --- a/charts/values.minikube.yaml +++ b/charts/values.minikube.yaml @@ -6,7 +6,7 @@ service: type: NodePort configmap: - DJANGO_SETTINGS_MODULE: api.settings.local + CONNEXION_SETTINGS_MODULE: api.settings.local RYR_API_API_OPTS: "--reload --timeout 1800 --chdir /usr/src/app" env: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 57014a4..305a0d2 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -15,12 +15,6 @@ DATE=$(date -u +%Y%m%dT%H%M%S%Z) # Run the API. if [ "$1" == "api" ]; then - # Prepare the static files. - django-admin collectstatic --noinput - - # Migrate db, so we have the latest db schema. - django-admin migrate - # Start WSGI server. exec gunicorn ${RYR_API_API_OPTS} --log-level ${RYR_LOG_LEVEL} -b 0.0.0.0:${RYR_API_API_PORT} api.wsgi fi diff --git a/docs/source/collectors/collectors.rst b/docs/source/collectors/collectors.rst index e6c2d79..570a007 100644 --- a/docs/source/collectors/collectors.rst +++ b/docs/source/collectors/collectors.rst @@ -4,14 +4,14 @@ collectors package Base collectors module ---------------------- -.. automodule:: api.apps.api.collectors.base +.. automodule:: api.collectors.base :members: :show-inheritance: Generic collectors module ------------------------- -.. automodule:: api.apps.api.collectors.generic +.. automodule:: api.collectors.generic :members: :undoc-members: :show-inheritance: @@ -19,7 +19,7 @@ Generic collectors module Google collectors module ------------------------ -.. automodule:: api.apps.api.collectors.google +.. automodule:: api.collectors.google :members: :undoc-members: :show-inheritance: @@ -27,7 +27,7 @@ Google collectors module Yelp collectors module ---------------------- -.. automodule:: api.apps.api.collectors.yelp +.. automodule:: api.collectors.yelp :members: :undoc-members: :show-inheritance: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 1a14c1a..ed6f1c3 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -176,7 +176,7 @@ components: type: string description: Business address example: '5, 48 Pirrama Rd, Pyrmont NSW 2009, Australia' - id: + place_id: type: string description: Business ID, specific to a particular collector example: ChIJyWEHuEmuEmsRm9hTkapTCrk diff --git a/requirements-dev.txt b/requirements-dev.txt index 79426fc..b07d7b3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,17 +1,17 @@ -Faker==0.9.1 -Sphinx==1.8.0 +Faker==0.9.2 +Sphinx==1.8.1 bpython==0.17.1 -flake8==3.5.0 +flake8==3.6.0 pydocstyle==2.1.1 pylint==2.1.1 model-mommy==1.6.0 pytest-cov==2.6.0 pytest-mock==1.10.0 pytest-socket==0.3.1 -pytest==3.8.0 +pytest==3.10.0 q==2.6 -responses==0.9.0 +responses==0.10.3 sphinxjp.themes.basicstrap==0.4.3 tox==3.3.0 -wheel==0.31.1 +wheel==0.32.2 yapf==0.24.0 diff --git a/requirements.txt b/requirements.txt index 80f4c2e..2cb5669 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,10 @@ git+https://github.com/celery/celery.git@master#egg=celery -Django==2.0.7 Pygments==2.2.0 -dataclasses==0.6 -dj-database-url==0.5.0 -django-cors-headers==2.4.0 -django-model-utils==3.1.2 -django-redis-cache==1.7.1 -djangorestframework==3.8.2 +connexion[swagger-ui]==2.0.1 googlemaps==3.0.2 gunicorn==19.9.0 json-tricks==3.12.2 -lxml>=4.2.3 -pbr>=4.1.0 -psycopg2-binary==2.7.5 +lxml==4.2.5 +pbr==5.1.1 redis==2.10.6 -requests==2.19.1 -whitenoise==3.3.1 +requests==2.20.1 diff --git a/setup.cfg b/setup.cfg index 5d952ff..5d0e05c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,6 +15,10 @@ classifier = Programming Language :: Python :: 3 Programming Language :: Python :: 3.6 +[files] +data_files = + etc/ryr-api = openapi/openapi.yaml + [entry_points] console_scripts = api = api.main:main diff --git a/api/apps/api/collectors/__init__.py b/tests/__init__.py similarity index 100% rename from api/apps/api/collectors/__init__.py rename to tests/__init__.py diff --git a/api/apps/api/tests/__init__.py b/tests/celery/__init__.py similarity index 100% rename from api/apps/api/tests/__init__.py rename to tests/celery/__init__.py diff --git a/api/celery/tests/test_tasks.py b/tests/celery/test_tasks.py similarity index 92% rename from api/celery/tests/test_tasks.py rename to tests/celery/test_tasks.py index fed37f6..5689e59 100644 --- a/api/celery/tests/test_tasks.py +++ b/tests/celery/test_tasks.py @@ -12,13 +12,13 @@ import responses from api.celery import tasks -from api.apps.api.collectors.base import BusinessInfo -from api.apps.api.collectors.base import PlaceSearchSummary -from api.apps.api.collectors.generic import CollectorClient -from api.apps.api.collectors.yelp import YelpCollector -from api.apps.api.tests.collectors.test_google import GOOGLE_MAPS_DETAILS_RESPONSE -from api.apps.api.tests.collectors.test_yelp import YELP_DETAILS_RESPONSE -from api.apps.api.tests.collectors.test_yelp import YELP_SEARCH_RESPONSE +from api.collectors.base import BusinessInfo +from api.collectors.base import PlaceSearchSummary +from api.collectors.generic import CollectorClient +from api.collectors.yelp import YelpCollector +from tests.collectors.test_google import GOOGLE_MAPS_DETAILS_RESPONSE +from tests.collectors.test_yelp import YELP_DETAILS_RESPONSE +from tests.collectors.test_yelp import YELP_SEARCH_RESPONSE class TestCeleryTasks: diff --git a/api/apps/api/tests/collectors/test_base.py b/tests/collectors/test_base.py similarity index 83% rename from api/apps/api/tests/collectors/test_base.py rename to tests/collectors/test_base.py index 183d987..87a5289 100644 --- a/api/apps/api/tests/collectors/test_base.py +++ b/tests/collectors/test_base.py @@ -2,9 +2,9 @@ from faker import Faker import pytest -from api.apps.api.collectors.base import AbstractCollector -from api.apps.api.collectors.base import BusinessInfo -from api.apps.api.collectors.base import PlaceSearchSummary +from api.collectors.base import AbstractCollector +from api.collectors.base import BusinessInfo +from api.collectors.base import PlaceSearchSummary class TestBusinessInfo: @@ -78,12 +78,12 @@ def test_to_json_00(self): """Ensure the object serializes to JSON correctly.""" b = BusinessInfo(name='name1', address='address2') actual = b.to_json(indent=None) - expected = '{"__instance_type__": ["api.apps.api.collectors.base", "BusinessInfo"], "attributes": {"address": "address2", "contact_name": "", "email": "", "extra_info": "", "latitude": 0.0, "longitude": 0.0, "name": "name1", "parking_info": "", "phone": "", "type": "", "website": "", "weight": 0}}' + expected = '{"__instance_type__": ["api.collectors.base", "BusinessInfo"], "attributes": {"address": "address2", "contact_name": "", "email": "", "extra_info": "", "latitude": 0.0, "longitude": 0.0, "name": "name1", "parking_info": "", "phone": "", "type": "", "website": "", "weight": 0}}' assert actual == expected def test_from_json_00(self): """Ensure the object get deserialized from JSON correctly.""" - json_object = '{"__instance_type__": ["api.apps.api.collectors.base", "BusinessInfo"], "attributes": {"address": "address2", "contact_name": "", "email": "", "extra_info": "", "latitude": 0.0, "longitude": 0.0, "name": "name1", "parking_info": "", "phone": "", "type": "", "website": "", "weight": 0}}' + json_object = '{"__instance_type__": ["api.collectors.base", "BusinessInfo"], "attributes": {"address": "address2", "contact_name": "", "email": "", "extra_info": "", "latitude": 0.0, "longitude": 0.0, "name": "name1", "parking_info": "", "phone": "", "type": "", "website": "", "weight": 0}}' actual = BusinessInfo.from_json(json_object) expected = BusinessInfo(name='name1', address='address2') assert actual == expected diff --git a/api/apps/api/tests/collectors/test_generic.py b/tests/collectors/test_generic.py similarity index 95% rename from api/apps/api/tests/collectors/test_generic.py rename to tests/collectors/test_generic.py index a1835fe..02c79f0 100644 --- a/api/apps/api/tests/collectors/test_generic.py +++ b/tests/collectors/test_generic.py @@ -3,8 +3,8 @@ from faker import Faker import pytest -from api.apps.api.collectors.base import PlaceSearchSummary -from api.apps.api.collectors.generic import CollectorClient +from api.collectors.base import PlaceSearchSummary +from api.collectors.generic import CollectorClient class TestCollectorClient: @@ -83,7 +83,7 @@ def test_lookup_place_02(self, mocker): fake_address = self.fake.address() c = CollectorClient(self.fake.pystr()) c.search_places = mocker.Mock() - c.retrieve_search_summary = mocker.Mock(return_value=PlaceSearchSummary(id=fake_place_id)) + c.retrieve_search_summary = mocker.Mock(return_value=PlaceSearchSummary(place_id=fake_place_id)) c.get_place_details = mocker.Mock() c.lookup_place(name=fake_name, address=fake_address) diff --git a/api/apps/api/tests/collectors/test_google.py b/tests/collectors/test_google.py similarity index 99% rename from api/apps/api/tests/collectors/test_google.py rename to tests/collectors/test_google.py index 635a496..bd4ce0a 100644 --- a/api/apps/api/tests/collectors/test_google.py +++ b/tests/collectors/test_google.py @@ -6,9 +6,9 @@ from faker import Faker import pytest -from api.apps.api.collectors.base import BusinessInfo -from api.apps.api.collectors.base import PlaceSearchSummary -from api.apps.api.collectors.google import GoogleCollector +from api.collectors.base import BusinessInfo +from api.collectors.base import PlaceSearchSummary +from api.collectors.google import GoogleCollector @pytest.fixture() @@ -113,7 +113,7 @@ def test_retrieve_search_summary_02(self, google_collector): gmaps.search_results = GOOGLE_MAPS_SEARCH_RESPONSE actual = gmaps.retrieve_search_summary() expected = PlaceSearchSummary( - id='ChIJyWEHuEmuEmsRm9hTkapTCrk', + place_id='ChIJyWEHuEmuEmsRm9hTkapTCrk', name='Rhythmboat Cruises', address='Pyrmont Bay Wharf Darling Dr, Sydney', ) diff --git a/api/apps/api/tests/collectors/test_yelp.py b/tests/collectors/test_yelp.py similarity index 97% rename from api/apps/api/tests/collectors/test_yelp.py rename to tests/collectors/test_yelp.py index bdddbe3..9344d9e 100644 --- a/api/apps/api/tests/collectors/test_yelp.py +++ b/tests/collectors/test_yelp.py @@ -6,9 +6,9 @@ import requests import pytest -from api.apps.api.collectors.base import BusinessInfo -from api.apps.api.collectors.base import PlaceSearchSummary -from api.apps.api.collectors.yelp import YelpCollector +from api.collectors.base import BusinessInfo +from api.collectors.base import PlaceSearchSummary +from api.collectors.yelp import YelpCollector class TestYelpCollector(): @@ -125,7 +125,7 @@ def test_retrieve_search_summary_02(self): yelp.search_results = YELP_SEARCH_RESPONSE actual = yelp.retrieve_search_summary() expected = PlaceSearchSummary( - id='four-barrel-coffee-san-francisco', + place_id='four-barrel-coffee-san-francisco', name='Four Barrel Coffee', address='', ) diff --git a/api/apps/api/tests/google_collector_details_epoch_coffee.json b/tests/google_collector_details_epoch_coffee.json similarity index 100% rename from api/apps/api/tests/google_collector_details_epoch_coffee.json rename to tests/google_collector_details_epoch_coffee.json diff --git a/api/apps/api/tests/google_collector_search_epoch_coffee.json b/tests/google_collector_search_epoch_coffee.json similarity index 100% rename from api/apps/api/tests/google_collector_search_epoch_coffee.json rename to tests/google_collector_search_epoch_coffee.json diff --git a/api/apps/api/tests/test_integration_collectors.py b/tests/test_integration_collectors.py similarity index 94% rename from api/apps/api/tests/test_integration_collectors.py rename to tests/test_integration_collectors.py index 1d8a7ed..d0940a6 100644 --- a/api/apps/api/tests/test_integration_collectors.py +++ b/tests/test_integration_collectors.py @@ -1,9 +1,9 @@ import json import os -from api.apps.api.collectors.generic import CollectorClient -from api.apps.api.collectors.google import GoogleCollector -from api.apps.api.collectors.base import PlaceSearchSummary +from api.collectors.generic import CollectorClient +from api.collectors.google import GoogleCollector +from api.collectors.base import PlaceSearchSummary def integration_test_yelp_search_place(): @@ -36,7 +36,7 @@ def integration_test_yelp_retrieve_place_details(): '221 West North Loop Boulevard, Austin', ) place_details = client.retrieve_place_details( - epoch_search_summary.id, + epoch_search_summary.place_id, epoch_search_summary.name, epoch_search_summary.address, ) @@ -89,7 +89,7 @@ def integration_test_full_google_yelp_workflow(): yelp = CollectorClient('yelp', api_key=yelp_api_key) yelp.authenticate() place_details = yelp.retrieve_place_details( - epoch_search_summary.id, + epoch_search_summary.place_id, epoch_search_summary.name, epoch_search_summary.address, ) @@ -116,7 +116,7 @@ def integration_test_full_google_yelp_workflow_with_generic(): # Lookup the place on Google Maps. google = CollectorClient('google', api_key=places_api_key) google.authenticate() - google_place_details = google.lookup_place(epoch_search_summary.id) + google_place_details = google.lookup_place(epoch_search_summary.place_id) gb = google.to_business_info() print('*** Google:') print(gb) diff --git a/tox.ini b/tox.ini index 1284bb1..80376c4 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ passenv = deps = -r{toxinidir}/requirements.txt {[testenv]deps} -commands = py.test -x --junitxml={env:CIRCLE_TEST_REPORTS:test-results}/pytest/result.xml --cov-report term-missing --cov-report html --cov-report xml --cov=api {toxinidir}/api/ {posargs} +commands = py.test -x --junitxml={env:CIRCLE_TEST_REPORTS:test-results}/pytest/result.xml --cov-report term-missing --cov-report html --cov-report xml --cov=api {toxinidir}/ {posargs} [testenv:docs] skip_install = false