diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..6851520a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,86 @@ +Dockerfile +.dockerignore + +.git/ + +# custom configs +custom.cfg +data_sources.cfg +custom_wps.cfg +Makefile.config + +# Python / Extensions etc. +**/*.mo +**/*.so +**/*.pyc +**/*.pyo +**/*.egg +**/*.egg-info +**/*.sqlite +**/*.bak +**/__pycache__ +**/*.py[cod] + +# Unit test / Coverage reports +**/*.cache +**/*.coverage +**/*.pytest_cache +**/*.tox +**/nosetests.xml +**/*.log +**/*.lock +tests/ +testdata.json + +# R +**/*.Rhistory + +# Eclipse / PyDev +**/*.project +**/*.pydevproject +**/*.settings + +# PyCharm +**/*.idea + +# Intellij +**/*.iml + +# Kate +**/*.kate-swp + +# Sublime Text Editor +**/*.sublime* + +# build / env config +bin/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +env/ +opt/ +parts/ +**/*.installed.cfg +**/*.bak.* + +# sphinx +docs/ + +# External Sources +src/ + +# IPython and Notebooks +.ipynb_checkpoints +**/*.ipynb + +# gcc/fortran +**/*.o +**/*.a +**/*.mod +**/*.out + +# process results +**/*.tif +**/*.zip diff --git a/.gitignore b/.gitignore index 9546db9e..b45dd9c4 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ __pycache__ .tox nosetests.xml unit_tests/testdata.json -build +coverage/ # R *.Rhistory @@ -63,3 +63,7 @@ testdata.json *.a *.mod *.out + +# Environments +env/ +venv/ diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a2da894..ffa88a86 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,3 +1,4 @@ Carsten Ehbrecht Eric Lemoine (http://github.com/elemoine/papyrus_ogcproxy) - +David Byrns +Francis Charette-Migneault diff --git a/CHANGES.rst b/CHANGES.rst index 5f9fd1b4..169177fe 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,18 @@ Changes ******* +Unreleased +========== + +New Features: + +* Provide a `Dockerfile` for building `Twitcher`. +* Improve `Makefile` conda environment setup/activation/update to speedup target operations when possible. +* Add `Makefile` targets for `docker`, `conda`, `bumpversion` and `coverage` analysis related tasks. +* Provide ``AdapterInterface`` to allow overriding store implementations with configuration setting ``twitcher.adapter``. +* Add version auto-update (number and date) of these 'changes' with ``bump2version``. +* Update requirements with missing dependencies when building docker image. + 0.4.0 (2019-05-02) ================== diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..da68b530 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +# vim:set ft=dockerfile: +FROM python:3.7-alpine +LABEL Description="Twitcher" Vendor="Birdhouse" Maintainer="https://github.com/bird-house/twitcher" + +# Configure hostname and ports for services +ENV HTTP_PORT 8080 +ENV HTTPS_PORT 8443 +ENV OUTPUT_PORT 8000 +ENV HOSTNAME localhost +EXPOSE 9001 $HTTP_PORT $HTTPS_PORT $OUTPUT_PORT + +ENV HOME /root +ENV TWITCHER_DIR /opt/birdhouse/src/twitcher +WORKDIR $TWITCHER_DIR + +# copy basic requirements/references and build dependencies +# will be skipped if only source code has been updated +COPY \ + requirements* \ + setup.py \ + README.rst \ + CHANGES.rst \ + $TWITCHER_DIR/ +COPY \ + twitcher/__init__.py \ + twitcher/__version__.py \ + $TWITCHER_DIR/twitcher/ +RUN apk update \ + && apk add \ + bash \ + libxslt-dev \ + libxml2 \ + libffi-dev \ + openssl-dev \ + && apk add --virtual .build-deps \ + python-dev \ + gcc \ + musl-dev \ + && pip install --no-cache-dir --upgrade pip setuptools \ + && pip install --no-cache-dir -e $TWITCHER_DIR \ + && apk --purge del .build-deps + +# copy source code and install it +COPY ./ $TWITCHER_DIR +RUN pip install --no-dependencies -e $TWITCHER_DIR + +CMD ["pserve", "development.ini"] diff --git a/Makefile b/Makefile index aa7a216c..dc7069ce 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,41 @@ +#!/usr/bin/env bash + # Application -APP_ROOT := $(CURDIR) +APP_ROOT := $(abspath $(lastword $(MAKEFILE_LIST))/..) APP_NAME := twitcher +INI_FILE ?= $(APP_ROOT)/development.ini + +# Conda +CONDA_HOME ?= $(shell conda info --base 2> /dev/null) +CONDA_ENV ?= $(APP_NAME) +CONDA_BIN := $(CONDA_HOME)/bin/conda +CONDA_TARGET_PREFIX := readlink -e "$(CONDA_HOME)/envs/$(CONDA_ENV)" 2> /dev/null +CONDA_ACTUAL_PREFIX := readlink -e "${CONDA_PREFIX}" 2> /dev/null +# don't activate and update conda env if it already is activated +# (avoids long conda 'solving environment' time) +ifeq ($(shell $(CONDA_TARGET_PREFIX)),$(shell $(CONDA_ACTUAL_PREFIX))) + CONDA_MSG := Conda environment already activated + CONDA_CMD := echo "$(CONDA_MSG)"; +else + CONDA_MSG := Will activate conda environment with 'CONDA_CMD' + CONDA_CMD := echo "Activating conda env \'$(CONDA_ENV)\' ..."; source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; +endif -# Anaconda -CONDA := $(shell command -v conda 2> /dev/null) -ANACONDA_HOME := $(shell conda info --base 2> /dev/null) -CONDA_ENV := $(APP_NAME) +DOCKER_TAG := birdhouse/twitcher:0.4.0 # Temp files TEMP_FILES := *.egg-info *.log *.sqlite +# Bumpversion 'dry' config +# if 'dry' is specified as target, any bumpversion call using 'BUMP_XARGS' will not apply changes +BUMP_XARGS ?= --verbose --allow-dirty +ifeq ($(filter dry, $(MAKECMDGOALS)), dry) + BUMP_XARGS := $(BUMP_XARGS) --dry-run +endif +.PHONY: dry +dry: setup.cfg + @-echo > /dev/null + # end of configuration .DEFAULT_GOAL := help @@ -19,74 +45,125 @@ all: help .PHONY: help help: - @echo "Please use \`make ' where is one of" - @echo " help to print this help message. (Default)" - @echo " install to install $(APP_NAME) by running 'python setup.py develop'." - @echo " db to upgrade or initialize database." - @echo " start to start $(APP_NAME) service as daemon (background process)." - @echo " clean to delete all files that are created by running buildout." + @echo "Please use 'make ' where is one of:" + @echo " help to print this help message. (Default)" + @echo " install to install $(APP_NAME) by running 'python setup.py develop'." + @echo " install-dev to execute 'install' with additional development requirements." + @echo " migrate to upgrade or initialize database." + @echo " start to start $(APP_NAME) service as daemon (background process)." + @echo " clean to delete all files that are created by running buildout." + @echo "\Environment targets:" + @echo " bootstrap to install $(APP_NAME) after updating the conda environment'." + @echo " bootstrap-dev to execute 'bootstrap' with additional development requirements." + @echo " conda-check to verify that conda is installed and found." + @echo " conda-update to update the conda environment with 'environment.yml' file." + @echo " conda-clean to remove the conda environment." + @echo " conda-spec to generate Conda specifications file." + @echo "\nDocker targets:" + @echo " docker-build to build the docker image with current code base and version." + @echo " docker-push to push the built docker image to the tagged repository." @echo "\nTesting targets:" - @echo " test to run tests (but skip long running tests)." - @echo " testall to run all tests (including long running tests)." - @echo " pep8 to run pep8 code style checks." + @echo " test to run tests (but skip long running tests)." + @echo " test-all to run all tests (including long running tests)." + @echo " pep8 to run pep8 code style checks." + @echo " converage-test to run all tests with coverage analysis." + @echo " converage-table to display tests coverage analysis in commandline table." + @echo " converage-html to generate an HTML report from tests coverage analysis." + @echo " converage-clean to delete files generated by tests coverage analysis." @echo "\nSphinx targets:" - @echo " docs to generate HTML documentation with Sphinx." + @echo " docs to generate HTML documentation with Sphinx." @echo "\nDeployment targets:" - @echo " spec to generate Conda spec file." + @echo " debug to print variable values employed by this Makefile." + @echo " bump to update the package version." + @echo " dry to only display results (not applied) when combined with 'bump'." + +.PHONY: debug +debug: + @-echo "Following variables are used:" + @-echo " SHELL: $(SHELL)" + @-echo " APP_ROOT: $(APP_ROOT)" + @-echo " APP_NAME: $(APP_NAME)" + @-echo " BUMP_XARGS: $(BUMP_XARGS)" + @-echo " CONDA_HOME: $(CONDA_HOME)" + @-echo " CONDA_BIN: $(CONDA_BIN)" + @-echo " CONDA_ENV: $(CONDA_ENV)" + @-echo " CONDA_MSG: $(CONDA_MSG)" + @-echo " CONDA_CMD: $(CONDA_CMD)" + @-echo " CONDA_TARGET_PREFIX [literal]: $(CONDA_TARGET_PREFIX)" + @-echo " CONDA_ACTUAL_PREFIX [called]: `$(CONDA_TARGET_PREFIX)`" + @-echo " CONDA_TARGET_PREFIX [literal]: $(CONDA_ACTUAL_PREFIX)" + @-echo " CONDA_ACTUAL_PREFIX [called]: `$(CONDA_ACTUAL_PREFIX)`" + @-echo " DOCKER_TAG: $(DOCKER_TAG)" ## Conda targets -.PHONY: check_conda -check_conda: -ifndef CONDA - $(error "Conda is not available. Please install miniconda: https://conda.io/miniconda.html") +.PHONY: conda-check +conda-check: +ifndef CONDA_BIN + $(error "Conda is not available. Please install miniconda: https://conda.io/miniconda.html") endif -.PHONY: conda_env -conda_env: check_conda +.PHONY: conda-update +conda-update: conda-check @echo "Updating conda environment $(CONDA_ENV) ..." - "$(CONDA)" env update -n $(CONDA_ENV) -f environment.yml + @-"$(CONDA_BIN)" env update -n "$(CONDA_ENV)" -f environment.yml -.PHONY: envclean -envclean: check_conda +.PHONY: conda-clean +conda-clean: conda-check @echo "Removing conda env $(CONDA_ENV)" - @-"$(CONDA)" remove -n $(CONDA_ENV) --yes --all + @-"$(CONDA_BIN)" remove -n "$(CONDA_ENV)" --yes --all -.PHONY: spec -spec: check_conda +.PHONY: conda-spec +spec: conda-check @echo "Updating conda environment specification file ..." - @-"$(CONDA)" list -n $(CONDA_ENV) --explicit > spec-file.txt + @-"$(CONDA_BIN)" list -n "$(CONDA_ENV)" --explicit > spec-file.txt + +## Version targets + +.PHONY: bump-dep +bump-dep: conda-check + @-bash -c '$(CONDA_CMD) pip install bump2version' + +.PHONY: bump +bump: conda-check bump-dep + @-echo "Updating package version ..." + @[ "${VERSION}" ] || ( echo ">> 'VERSION' is not set"; exit 1 ) + @-bash -c '$(CONDA_CMD) bump2version $(BUMP_XARGS) --new-version "${VERSION}" patch;' ## Build targets .PHONY: bootstrap -bootstrap: check_conda conda_env bootstrap_dev +bootstrap: conda-check conda-update @echo "Bootstrap ..." -.PHONY: bootstrap_dev -bootstrap_dev: - @echo "Installing development requirements for tests and docs ..." - @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && pip install -r requirements_dev.txt" +.PHONY: bootstrap-dev +bootstrap-dev: conda-check conda-update + @echo "Bootstrap for development ..." .PHONY: install -install: bootstrap +install: conda-check bootstrap @echo "Installing application ..." - @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && python setup.py develop" + @-bash -c '$(CONDA_CMD) pip install -e "$(APP_ROOT)"' @echo "\nStart service with \`make start'" +.PHONY: install-dev +install-dev: conda-check bootstrap-dev + @echo "Installing development requirements for tests and docs ..." + @-bash -c '$(CONDA_CMD) pip install -e "$(APP_ROOT)[dev]"' + .PHONY: db -db: +db: conda-check @echo "Upgrade or initialize database ..." - @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && alembic -c development.ini upgrade head" - @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && initialize_twitcher_db development.ini" + @-bash -c '$(CONDA_CMD) alembic -c "$(INI_FILE)" upgrade head' + @-bash -c '$(CONDA_CMD) initialize_twitcher_db "$(INI_FILE)"' .PHONY: start -start: check_conda +start: conda-check @echo "Starting application ..." - @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && pserve development.ini &" + @-bash -c '$(CONDA_CMD) pserve "$(INI_FILE)" &' .PHONY: clean -clean: srcclean envclean +clean: srcclean conda-clean @echo "Cleaning generated files ..." @-for i in $(TEMP_FILES); do \ test -e $$i && rm -v -rf $$i; \ @@ -95,7 +172,7 @@ clean: srcclean envclean .PHONY: srcclean srcclean: @echo "Removing *.pyc files ..." - @-find $(APP_ROOT) -type f -name "*.pyc" -print | xargs rm + @-find "$(APP_ROOT)" -type f -name "*.pyc" -print | xargs rm .PHONY: distclean distclean: clean @@ -103,27 +180,67 @@ distclean: clean @git diff --quiet HEAD || echo "There are uncommited changes! Not doing 'git clean' ..." @-git clean -dfx +## Docker targets + +.PHONY: docker-build +docker-build: + @echo "Building docker image: $(DOCKER_TAG)" + @-docker build "$(APP_ROOT)" -t "$(DOCKER_TAG)" + +.PHONY: docker-push +docker-push: + @echo "Pushing docker image: $(DOCKER_TAG)" + @-docker push "$(DOCKER_TAG)" + ## Test targets .PHONY: test -test: check_conda +test: conda-check @echo "Running tests (skip slow and online tests) ..." - @bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);pytest -v -m 'not slow and not online' tests/" + @bash -c '$(CONDA_CMD) pytest -v -m "not slow and not online" "$(APP_ROOT)/tests/"' -.PHONY: testall -testall: check_conda +.PHONY: test-unit +test-unit: conda-check + @echo "Running only unit tests ..." + @bash -c '$(CONDA_CMD) pytest -v -m "not mongo and not slow and not online" "$(APP_ROOT)/tests/"' + +.PHONY: test-all +test-all: conda-check @echo "Running all tests (including slow and online tests) ..." - @bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && pytest -v tests/" + @bash -c '$(CONDA_CMD) pytest -v "$(APP_ROOT)/tests/"' .PHONY: pep8 -pep8: check_conda +pep8: conda-check @echo "Running pep8 code style checks ..." - @bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && flake8" + @bash -c '$(CONDA_CMD) flake8' + +# run only if .coverage doesn't already exist +# all other coverage targets will use existing results if available +.coverage: + @echo "Running coverage analysis..." + @bash -c '$(CONDA_CMD) pytest --cov "$(APP_ROOT)/tests/"' + +.PHONY: coverage-test +coverage-test: coverage-clean .coverage + +.PHONY: coverage-clean +coverage-clean: + @-rm -f .coverage + @-rm -fr ./coverage/ + +.PHONY: coverage-table +coverage-table: .coverage + @bash -c '$(CONDA_CMD) coverage report -m' + +.PHONY: coverage-html +coverage-html: .coverage + @bash -c '$(CONDA_CMD) coverage html -d "$(APP_ROOT)/coverage"' + @-echo "Coverage report: open file://$(APP_ROOT)/coverage/index.html" ## Sphinx targets .PHONY: docs -docs: check_conda +docs: conda-check @echo "Generating docs with Sphinx ..." - @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);$(MAKE) -C $@ clean html" - @echo "open your browser: open docs/build/html/index.html" + @-bash -c '$(CONDA_CMD) $(MAKE) -C $@ clean html' + @echo "open your browser: open file://$(APP_ROOT)/docs/build/html/index.html" diff --git a/development.ini b/development.ini index 88383854..4eec7468 100644 --- a/development.ini +++ b/development.ini @@ -24,15 +24,16 @@ retry.attempts = 3 # twitcher twitcher.url = http://localhost:8000 -twitcher.rcpinterface = true +twitcher.adapter = default +twitcher.rpcinterface = true twitcher.username = twitcher.password = twitcher.ows_security = true twitcher.ows_proxy = true twitcher.ows_proxy_delegate = false +twitcher.ows_proxy_protected_path = /ows twitcher.workdir = twitcher.prefix = -twitcher.ows_proxy_protected_path = /ows ### # wsgi server configuration diff --git a/requirements.txt b/requirements.txt index 8823cc11..7830770c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,8 @@ requests requests_oauthlib argcomplete pytz +lxml +pyopenssl # debug pyramid_debugtoolbar # deploy diff --git a/requirements_dev.txt b/requirements_dev.txt index 0a2d06fa..2da59078 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -6,4 +6,4 @@ flake8 # docs sphinx>=1.7 # release -bumpversion +bump2version diff --git a/setup.cfg b/setup.cfg index 9a83f56e..befd7a04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,8 +3,20 @@ current_version = 0.4.0 commit = True tag = True -[metadata] -description-file = README.rst +[bumpversion:file:CHANGES.rst] +search = + Unreleased + ========== +replace = + Unreleased + ========== + + {new_version} ({now:%%Y-%%m-%%d}) + ================== + +[bumpversion:file:Makefile] +search = DOCKER_TAG := birdhouse/twitcher:{current_version} +replace = DOCKER_TAG := birdhouse/twitcher:{new_version} [bumpversion:file:twitcher/__version__.py] search = __version__ = '{current_version}' @@ -14,6 +26,9 @@ replace = __version__ = '{new_version}' search = release = '{current_version}' replace = release = '{new_version}' +[metadata] +description-file = README.rst + [tool:pytest] addopts = --strict diff --git a/setup.py b/setup.py index 9a0070c4..e4eac350 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ exec(f.read(), about) reqs = [line.strip() for line in open('requirements.txt')] -extra_reqs = [line.strip() for line in open('requirements_dev.txt')] +dev_reqs = [line.strip() for line in open('requirements_dev.txt')] setup(name='pyramid_twitcher', version=about['__version__'], @@ -33,7 +33,7 @@ zip_safe=False, test_suite='twitcher', install_requires=reqs, - extra_requires=extra_reqs, + extra_requires={"dev": dev_reqs}, # pip install ".[dev]" entry_points="""\ [paste.app_factory] main = twitcher:main diff --git a/tests/test_adapter.py b/tests/test_adapter.py new file mode 100644 index 00000000..155a9ee6 --- /dev/null +++ b/tests/test_adapter.py @@ -0,0 +1,68 @@ +from twitcher.adapter import import_adapter, get_adapter_factory, TWITCHER_ADAPTER_DEFAULT +from twitcher.adapter.base import AdapterInterface +from twitcher.adapter.default import DefaultAdapter +from twitcher.store import ServiceStoreInterface +from pyramid.testing import DummyRequest +import pytest + + +def test_import_adapter(): + adapter = import_adapter('twitcher.adapter.default.DefaultAdapter') + assert adapter is DefaultAdapter, "Expect {!s}, but got {!s}".format(DefaultAdapter, adapter) + assert isinstance(adapter({}), AdapterInterface), \ + "Expect {!s}, but got {!s}".format(AdapterInterface, type(adapter)) + + +def test_adapter_factory_default_explicit(): + settings = {'twitcher.adapter': TWITCHER_ADAPTER_DEFAULT} + adapter = get_adapter_factory(settings) + assert isinstance(adapter, DefaultAdapter), "Expect {!s}, but got {!s}".format(DefaultAdapter, type(adapter)) + + +def test_adapter_factory_none_specified(): + adapter = get_adapter_factory({}) + assert isinstance(adapter, DefaultAdapter), "Expect {!s}, but got {!s}".format(DefaultAdapter, type(adapter)) + + +# noinspection PyAbstractClass +class TestAdapter(AdapterInterface): + def servicestore_factory(self, request): + class DummyServiceStore(ServiceStoreInterface): + def save_service(self, service): return True # noqa: E704 + def delete_service(self, service): pass # noqa: E704 + def list_services(self): return ["test"] # noqa: E704 + def fetch_by_name(self, name): return name # noqa: E704 + def fetch_by_url(self, url): return url # noqa: E704 + def clear_services(self): pass # noqa: E704 + return DummyServiceStore(request) + + +# noinspection PyPep8Naming +def test_adapter_factory_TestAdapter_valid_import(): + settings = {'twitcher.adapter': '{}.{}'.format(TestAdapter.__module__, TestAdapter.__name__)} + adapter = get_adapter_factory(settings) + assert isinstance(adapter, TestAdapter), "Expect {!s}, but got {!s}".format(TestAdapter, type(adapter)) + + +# noinspection PyAbstractClass +class TestAdapterFake(object): + pass + + +# noinspection PyPep8Naming +def test_adapter_factory_TestAdapter_invalid_raised(): + settings = {'twitcher.adapter': '{}.{}'.format(TestAdapterFake.__module__, TestAdapterFake.__name__)} + with pytest.raises(TypeError) as err: + get_adapter_factory(settings) + pytest.fail(msg="Invalid adapter not inheriting from 'AdapterInterface' should raise on import.") + adapter_str = '{}.{}'.format(AdapterInterface.__module__, AdapterInterface.__name__) + assert adapter_str in str(err), "Expected to have full adapter import string in error message." + + +# noinspection PyTypeChecker +def test_adapter_factory_call_servicestore_factory(): + settings = {'twitcher.adapter': '{}.{}'.format(TestAdapter.__module__, TestAdapter.__name__)} + adapter = get_adapter_factory(settings) + store = adapter.servicestore_factory(DummyRequest()) + assert isinstance(store, ServiceStoreInterface) + assert store.fetch_by_name("test") == "test", "Requested adapter with corresponding store should have been called." diff --git a/tests/test_owsproxy.py b/tests/test_owsproxy.py index 3b65e7e7..ca1bc0c9 100644 --- a/tests/test_owsproxy.py +++ b/tests/test_owsproxy.py @@ -13,7 +13,7 @@ from twitcher.owsexceptions import OWSAccessFailed from twitcher import owsproxy -from twitcher.owsproxy import owsproxy as owsproxy_view +from twitcher.owsproxy import owsproxy_view as owsproxy_view class OWSProxyTests(unittest.TestCase): diff --git a/twitcher/adapter/__init__.py b/twitcher/adapter/__init__.py new file mode 100644 index 00000000..335b03ca --- /dev/null +++ b/twitcher/adapter/__init__.py @@ -0,0 +1,96 @@ +from twitcher.adapter.default import DefaultAdapter, AdapterInterface +from twitcher.utils import get_settings + +from inspect import isclass +from typing import TYPE_CHECKING + +import logging +LOGGER = logging.getLogger("TWITCHER") + +if TYPE_CHECKING: + from twitcher.store import AccessTokenStoreInterface, ServiceStoreInterface + from twitcher.owssecurity import OWSSecurityInterface + from twitcher.typedefs import AnySettingsContainer + from pyramid.request import Request + from typing import AnyStr, Type, Union + + +TWITCHER_ADAPTER_DEFAULT = 'default' + + +def import_adapter(name): + # type: (AnyStr) -> Type[AdapterInterface] + """Attempts import of the class specified by python string ``package.module.class``.""" + components = name.split('.') + mod_name = components[0] + mod = __import__(mod_name) + for comp in components[1:]: + if not hasattr(mod, comp): + mod_name = '{mod}.{sub}'.format(mod=mod_name, sub=comp) + mod = __import__(mod_name, fromlist=[mod_name]) + continue + mod = getattr(mod, comp) + if not isclass(mod) or not issubclass(mod, AdapterInterface): + raise TypeError("Invalid reference is not of type '{}.{}'." + .format(AdapterInterface.__module__, AdapterInterface.__name__)) + return mod + + +def get_adapter_type(container): + # type: (AnySettingsContainer) -> AnyStr + """Finds the specified adapter from configuration settings.""" + settings = get_settings(container) + return str(settings.get('twitcher.adapter', TWITCHER_ADAPTER_DEFAULT.lower())) + + +def get_adapter_factory(container): + # type: (AnySettingsContainer) -> AdapterInterface + """ + Creates an adapter interface according to `twitcher.adapter` setting. + By default the :class:`twitcher.adapter.default.DefaultAdapter` implementation will be used. + """ + adapter_type = get_adapter_type(container) + if adapter_type != TWITCHER_ADAPTER_DEFAULT: + try: + adapter_class = import_adapter(adapter_type) + except Exception as e: + LOGGER.error("Adapter '{!s}' raised an exception during import : '{!r}'".format(adapter_type, e)) + raise + try: + LOGGER.info("Using adapter: '{!r}'".format(adapter_class)) + return adapter_class(container) + except Exception as e: + LOGGER.error("Adapter '{!s}' raised an exception during instantiation : '{!r}'".format(adapter_type, e)) + raise + return DefaultAdapter(container) + + +def get_adapter_store_factory( + adapter, # type: AdapterInterface + store_name, # type: AnyStr + request, # type: Request +): # type: (...) -> Union[AccessTokenStoreInterface, ServiceStoreInterface, OWSSecurityInterface] + """ + Retrieves the adapter store by name if it is defined. + + If another adapter than :class:`twitcher.adapter.default.DefaultAdapter` is provided, and that the store + cannot be found with it, `DefaultAdapter` is used as fallback to find the "default" store implementation. + + :returns: found store. + :raises NotImplementedError: when the store is not available from the adapter. + :raises Exception: when store instance was found but generated an error on creation. + """ + try: + store = getattr(adapter, store_name) + return store(request) + except NotImplementedError: + if isinstance(adapter, DefaultAdapter): + LOGGER.exception("Adapter 'DefaultAdapter' doesn't implement '{!r}', no way to recover.".format(store_name)) + raise + LOGGER.warning("Adapter '{!r}' doesn't implement '{!r}', falling back to 'DefaultAdapter' implementation." + .format(adapter, store_name)) + return get_adapter_store_factory(DefaultAdapter(request), store_name, request) + except Exception as e: + LOGGER.error("Adapter '{!r}' raised an exception while instantiating '{!r}' : '{!r}'" + .format(adapter, store_name, e)) + raise diff --git a/twitcher/adapter/base.py b/twitcher/adapter/base.py new file mode 100644 index 00000000..3a4dd16d --- /dev/null +++ b/twitcher/adapter/base.py @@ -0,0 +1,60 @@ +from twitcher.utils import get_settings + +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from twitcher.typedefs import AnySettingsContainer, JSON + from twitcher.store import AccessTokenStoreInterface, ServiceStoreInterface + from twitcher.owssecurity import OWSSecurityInterface + from pyramid.config import Configurator + from pyramid.request import Request + + +class AdapterInterface(object): + """ + Common interface allowing functionality overriding using an adapter implementation. + """ + def __init__(self, container): + # type: (AnySettingsContainer) -> None + self.settings = get_settings(container) + + def describe_adapter(self): + # type: () -> JSON + """ + Returns a JSON serializable dictionary describing the adapter implementation. + """ + raise NotImplementedError + + def configurator_factory(self, request): + # type: (Request) -> Configurator + """ + Returns the 'configurator' implementation of the adapter. + """ + raise NotImplementedError + + def tokenstore_factory(self, request): + # type: (Request) -> AccessTokenStoreInterface + """ + Returns the 'tokenstore' implementation of the adapter. + """ + raise NotImplementedError + + def servicestore_factory(self, request): + # type: (Request) -> ServiceStoreInterface + """ + Returns the 'servicestore' implementation of the adapter. + """ + raise NotImplementedError + + def owssecurity_factory(self, request): + # type: (Request) -> OWSSecurityInterface + """ + Returns the 'owssecurity' implementation of the adapter. + """ + raise NotImplementedError + + def owsproxy_config(self, request): + # type: (Request) -> None + """ + Returns the 'owsproxy' implementation of the adapter. + """ + raise NotImplementedError diff --git a/twitcher/adapter/default.py b/twitcher/adapter/default.py new file mode 100644 index 00000000..739dcdb5 --- /dev/null +++ b/twitcher/adapter/default.py @@ -0,0 +1,36 @@ +""" +Factories to create storage backends. +""" + +from twitcher.adapter.base import AdapterInterface +from twitcher.store import AccessTokenStore, ServiceStore +from twitcher.owssecurity import OWSSecurity +from twitcher.utils import get_settings + +from pyramid.config import Configurator + + +class DefaultAdapter(AdapterInterface): + def describe_adapter(self): + from twitcher.__version__ import __version__ + return {"name": "default", "version": str(__version__)} + + def configurator_factory(self, request): + settings = get_settings(request) + return Configurator(settings=settings) + + def tokenstore_factory(self, request): + return AccessTokenStore(request) + + def servicestore_factory(self, request): + return ServiceStore(request) + + def owssecurity_factory(self, request): + token_store = self.tokenstore_factory(request) + service_store = self.servicestore_factory(request) + return OWSSecurity(token_store, service_store) + + def owsproxy_config(self, request): + from twitcher.owsproxy import owsproxy_defaultconfig + config = self.configurator_factory(request) + owsproxy_defaultconfig(config) diff --git a/twitcher/frontpage.py b/twitcher/frontpage.py index df10eb89..b1ad0cb7 100644 --- a/twitcher/frontpage.py +++ b/twitcher/frontpage.py @@ -3,7 +3,7 @@ @view_config(route_name='frontpage', renderer='json') def frontpage(request): - return {'message': 'hello'} + return {'message': 'Twitcher Frontpage'} def includeme(config): diff --git a/twitcher/owsproxy.py b/twitcher/owsproxy.py index 06ea0857..6065a6b7 100644 --- a/twitcher/owsproxy.py +++ b/twitcher/owsproxy.py @@ -10,14 +10,20 @@ from pyramid.response import Response from pyramid.settings import asbool +from typing import TYPE_CHECKING +from twitcher.adapter import get_adapter_factory from twitcher.owsexceptions import OWSAccessForbidden, OWSAccessFailed -from twitcher.utils import replace_caps_url -from twitcher.store import ServiceStore +from twitcher.utils import replace_caps_url, get_settings, get_twitcher_url import logging LOGGER = logging.getLogger('TWITCHER') +if TYPE_CHECKING: + from twitcher.typedefs import AnySettingsContainer # noqa: F401 + from pyramid.config import Configurator # noqa: F401 + from typing import AnyStr # noqa: F401 + allowed_content_types = ( "application/xml", # XML @@ -46,8 +52,8 @@ ) -# requests.models.Reponse defaults its chunk size to 128 bytes, which is very slow -class BufferedResponse(): +# requests.models.Response defaults its chunk size to 128 bytes, which is very slow +class BufferedResponse(object): def __init__(self, resp): self.resp = resp @@ -79,9 +85,9 @@ def _send_request(request, service, extra_path=None, request_params=None): return OWSAccessFailed("Request failed: {}".format(e)) # Headers meaningful only for a single transport-level connection - HopbyHop = ['Connection', 'Keep-Alive', 'Public', 'Proxy-Authenticate', 'Transfer-Encoding', 'Upgrade'] + hop_by_hop = ['Connection', 'Keep-Alive', 'Public', 'Proxy-Authenticate', 'Transfer-Encoding', 'Upgrade'] return Response(app_iter=BufferedResponse(resp_iter), - headers={k: v for k, v in list(resp_iter.headers.items()) if k not in HopbyHop}) + headers={k: v for k, v in list(resp_iter.headers.items()) if k not in hop_by_hop}) else: try: resp = requests.request(method=request.method.upper(), url=url, data=request.body, headers=h, @@ -130,14 +136,28 @@ def _send_request(request, service, extra_path=None, request_params=None): return Response(content, status=resp.status_code, headers=headers) -def owsproxy(request): +def owsproxy_base_path(container): + # type: (AnySettingsContainer) -> AnyStr + settings = get_settings(container) + return settings.get('twitcher.ows_proxy_protected_path', '/ows').rstrip('/').strip() + + +def owsproxy_base_url(container): + # type: (AnySettingsContainer) -> AnyStr + twitcher_url = get_twitcher_url(container) + owsproxy_path = owsproxy_base_path(container) + return twitcher_url + owsproxy_path + + +def owsproxy_view(request): """ TODO: use ows exceptions """ try: service_name = request.matchdict.get('service_name') extra_path = request.matchdict.get('extra_path') - store = ServiceStore(request) + adapter = get_adapter_factory(request) + store = adapter.servicestore_factory(request) service = store.fetch_by_name(service_name) except Exception as err: # TODO: Store impl should raise appropriate exception like not authorized @@ -146,7 +166,7 @@ def owsproxy(request): return _send_request(request, service, extra_path, request_params=request.query_string) -def owsproxy_delegate(request): +def owsproxy_delegate_view(request): """ Delegates owsproxy request to external twitcher service. """ @@ -167,25 +187,31 @@ def owsproxy_delegate(request): return Response(resp.content, status=resp.status_code, headers=resp.headers) -def includeme(config): - settings = config.registry.settings - protected_path = settings.get('twitcher.ows_proxy_protected_path', '/ows') +def owsproxy_defaultconfig(config): + # type: (Configurator) -> None + settings = get_settings(config) if asbool(settings.get('twitcher.ows_proxy', True)): + protected_path = owsproxy_base_path(settings) LOGGER.debug('Twitcher {}/proxy enabled.'.format(protected_path)) config.add_route('owsproxy', protected_path + '/proxy/{service_name}') - # TODO: maybe configure extra path config.add_route('owsproxy_extra', protected_path + '/proxy/{service_name}/{extra_path:.*}') config.add_route('owsproxy_secured', protected_path + '/proxy/{service_name}/{access_token}') # use delegation mode? if asbool(settings.get('twitcher.ows_proxy_delegate', False)): LOGGER.debug('Twitcher {}/proxy delegation mode enabled.'.format(protected_path)) - config.add_view(owsproxy_delegate, route_name='owsproxy') - config.add_view(owsproxy_delegate, route_name='owsproxy_secured') + config.add_view(owsproxy_delegate_view, route_name='owsproxy') + config.add_view(owsproxy_delegate_view, route_name='owsproxy_secured') else: - # include twitcher config config.include('twitcher.config') - config.add_view(owsproxy, route_name='owsproxy') - config.add_view(owsproxy, route_name='owsproxy_secured') - config.add_view(owsproxy, route_name='owsproxy_extra') + # include mongodb + # config.include('twitcher.db') + config.add_view(owsproxy_view, route_name='owsproxy') + config.add_view(owsproxy_view, route_name='owsproxy_secured') + config.add_view(owsproxy_view, route_name='owsproxy_extra') + + +def includeme(config): + from twitcher.adapter import get_adapter_factory + get_adapter_factory(config).owsproxy_config(config) diff --git a/twitcher/owssecurity.py b/twitcher/owssecurity.py index 57908cf2..ebd48d02 100644 --- a/twitcher/owssecurity.py +++ b/twitcher/owssecurity.py @@ -1,33 +1,31 @@ -import tempfile - from twitcher.exceptions import AccessTokenNotFound, ServiceNotFound from twitcher.owsexceptions import OWSAccessForbidden, OWSInvalidParameterValue -from twitcher.store import AccessTokenStore, ServiceStore -from twitcher import datatype from twitcher.utils import path_elements, parse_service_name from twitcher.owsrequest import OWSRequest -from twitcher.esgf import fetch_certificate, ESGF_CREDENTIALS import logging LOGGER = logging.getLogger("TWITCHER") -def owssecurity_factory(request): - return OWSSecurity(AccessTokenStore(request), ServiceStore(request)) - - def verify_cert(request): if not request.headers.get('X-Ssl-Client-Verify', '') == 'SUCCESS': raise OWSAccessForbidden("A valid X.509 client certificate is needed.") -class OWSSecurity(object): +class OWSSecurityInterface(object): + + def check_request(self, request): + raise NotImplementedError + + +class OWSSecurity(OWSSecurityInterface): def __init__(self, tokenstore, servicestore): self.tokenstore = tokenstore self.servicestore = servicestore - def get_token_param(self, request): + @staticmethod + def get_token_param(request): token = None if 'token' in request.params: token = request.params['token'] # in params diff --git a/twitcher/rpcinterface.py b/twitcher/rpcinterface.py index cc7f0118..34c6b155 100644 --- a/twitcher/rpcinterface.py +++ b/twitcher/rpcinterface.py @@ -1,11 +1,10 @@ from pyramid.view import view_defaults -from pyramid_rpc.xmlrpc import xmlrpc_method from pyramid.settings import asbool from twitcher.api import ITokenManager, TokenManager from twitcher.api import IRegistry, Registry +from twitcher.adapter import get_adapter_factory from twitcher.tokengenerator import tokengenerator_factory -from .store import AccessTokenStore, ServiceStore import logging LOGGER = logging.getLogger("TWITCHER") @@ -15,10 +14,12 @@ class RPCInterface(ITokenManager, IRegistry): def __init__(self, request): self.request = request + self.adapter = get_adapter_factory(request) self.tokenmgr = TokenManager( tokengenerator_factory(request), - AccessTokenStore(request)) - self.srvreg = Registry(ServiceStore(request)) + self.adapter.tokenstore_factory(request)) + self.srvreg = Registry( + self.adapter.servicestore_factory(request)) def generate_token(self, valid_in_hours=1): """ diff --git a/twitcher/store.py b/twitcher/store.py index 53161742..d9ce6b2f 100644 --- a/twitcher/store.py +++ b/twitcher/store.py @@ -1,5 +1,11 @@ """ Read or write data from database. + +Stores should not be accessed directly, but instead should use the adapter interface. + +See also: + - :class:`twitcher.adapter.base.AdapterInterface` + - :func:`twitcher.adapter.get_adapter_factory` """ from twitcher.exceptions import ( @@ -7,22 +13,45 @@ ServiceNotFound, DatabaseError ) -from twitcher import namesgenerator from twitcher.utils import baseurl from twitcher import datatype from twitcher import models +from typing import TYPE_CHECKING from sqlalchemy.exc import DBAPIError +if TYPE_CHECKING: + from twitcher.models.token import AccessToken + from twitcher.models.service import Service + from pyramid.request import Request + from typing import AnyStr, List + + +class AccessTokenStoreInterface(object): + def __init__(self, request): # type: (Request) -> None + self.request = request + + def save_token(self, access_token): # type: (AccessToken) -> None + raise NotImplementedError -class AccessTokenStore(object): + def delete_token(self, token): # type: (AccessToken) -> None + raise NotImplementedError + + def fetch_by_token(self, token): # type: (AnyStr) -> None + raise NotImplementedError + + def clear_tokens(self): # type: () -> None + raise NotImplementedError + + +class AccessTokenStore(AccessTokenStoreInterface): """ Stores tokens in sql database. TODO: handle exceptions. """ def __init__(self, request): - self.request = request + super(AccessTokenStore, self).__init__(request) def save_token(self, access_token): """ @@ -77,12 +106,35 @@ def clear_tokens(self): raise DatabaseError -class ServiceStore(object): +class ServiceStoreInterface(object): + def __init__(self, request): # type: (Request) -> None + self.request = request + + def save_service(self, service): # type: (Service) -> None + raise NotImplementedError + + def delete_service(self, name): # type: (AnyStr) -> None + raise NotImplementedError + + def list_services(self): # type: () -> List[Service] + raise NotImplementedError + + def fetch_by_name(self, name): # type: (AnyStr) -> Service + raise NotImplementedError + + def fetch_by_url(self, url): # type: (AnyStr) -> Service + raise NotImplementedError + + def clear_services(self): # type: () -> None + raise NotImplementedError + + +class ServiceStore(ServiceStoreInterface): """ Stores a services. It inserts or updates the service with a given name. """ def __init__(self, request): - self.request = request + super(ServiceStore, self).__init__(request) def save_service(self, service): """ diff --git a/twitcher/tweens.py b/twitcher/tweens.py index 08252c1e..60ceceee 100644 --- a/twitcher/tweens.py +++ b/twitcher/tweens.py @@ -1,8 +1,8 @@ from pyramid.settings import asbool from pyramid.tweens import EXCVIEW +from twitcher.adapter import get_adapter_factory from twitcher.owsexceptions import OWSException, OWSNoApplicableCode -from twitcher.owssecurity import owssecurity_factory import logging LOGGER = logging.getLogger("TWITCHER") @@ -22,7 +22,8 @@ def ows_security_tween_factory(handler, registry): def ows_security_tween(request): try: - security = owssecurity_factory(request) + adapter = get_adapter_factory(request) + security = adapter.owssecurity_factory(request) security.check_request(request) return handler(request) except OWSException as err: diff --git a/twitcher/typedefs.py b/twitcher/typedefs.py new file mode 100644 index 00000000..037f9ce4 --- /dev/null +++ b/twitcher/typedefs.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from typing import AnyStr, Dict, List, Tuple, Union + from requests.structures import CaseInsensitiveDict + from pyramid.config import Configurator + from pyramid.registry import Registry + from pyramid.request import Request as PyramidRequest + from pyramid.response import Response as PyramidResponse + from webob.response import Response as WebobResponse + from webob.headers import ResponseHeaders, EnvironHeaders + from webtest.response import TestResponse + + Number = Union[int, float] + AnyValue = Union[AnyStr, Number, bool, None] + AnyKey = Union[AnyStr, int] + JSON = Dict[AnyKey, Union[AnyValue, Dict[AnyKey, 'JSON'], List['JSON']]] + + AnyContainer = Union[Configurator, Registry, PyramidRequest] + SettingValue = Union[AnyStr, Number, bool, None] + SettingsType = Dict[AnyStr, SettingValue] + AnySettingsContainer = Union[AnyContainer, SettingsType] + + CookiesType = Dict[AnyStr, AnyStr] + HeadersType = Dict[AnyStr, AnyStr] + CookiesTupleType = List[Tuple[AnyStr, AnyStr]] + HeadersTupleType = List[Tuple[AnyStr, AnyStr]] + CookiesBaseType = Union[CookiesType, CookiesTupleType] + HeadersBaseType = Union[HeadersType, HeadersTupleType] + OptionalHeaderCookiesType = Union[Tuple[None, None], Tuple[HeadersBaseType, CookiesBaseType]] + AnyHeadersContainer = Union[HeadersBaseType, ResponseHeaders, EnvironHeaders, CaseInsensitiveDict] + AnyCookiesContainer = Union[CookiesBaseType, PyramidRequest, AnyHeadersContainer] + AnyResponseType = Union[WebobResponse, PyramidResponse, TestResponse] diff --git a/twitcher/utils.py b/twitcher/utils.py index 3f681bed..0cb02f26 100644 --- a/twitcher/utils.py +++ b/twitcher/utils.py @@ -1,16 +1,45 @@ -import time +from pyramid.config import Configurator +from pyramid.request import Request +from pyramid.registry import Registry from datetime import datetime +from urllib import parse as urlparse +from lxml import etree +from typing import TYPE_CHECKING +import time import pytz import re -from lxml import etree from twitcher.exceptions import ServiceNotFound -from urllib import parse as urlparse - import logging LOGGER = logging.getLogger("TWITCHER") +if TYPE_CHECKING: + from twitcher.typedefs import AnySettingsContainer, SettingsType + from typing import AnyStr, Optional + + +def get_settings(container): + # type: (AnySettingsContainer) -> Optional[SettingsType] + """ + Retrieves the application ``settings`` from various containers referencing to it. + + :raises TypeError: if the container type cannot be identified to retrieve settings. + """ + if isinstance(container, (Configurator, Request)): + container = container.registry + if isinstance(container, Registry): + container = container.settings + if isinstance(container, dict): + return container + raise TypeError("Could not retrieve settings from container object of type [{}]".format(type(container))) + + +def get_twitcher_url(container): + # type: (AnySettingsContainer) -> AnyStr + settings = get_settings(container) + return settings.get('twitcher.url').rstrip('/').strip() + def sanitize(name, minlen=2, maxlen=25): """Lower-case name and replace all non-ascii chars by `_`.""" @@ -57,7 +86,7 @@ def expires_at(hours=1): def localize_datetime(dt, tz_name='UTC'): - """Provide a timzeone-aware object for a given datetime and timezone name + """Provide a timezone-aware object for a given datetime and timezone name """ tz_aware_dt = dt if dt.tzinfo is None: