diff --git a/.gitignore b/.gitignore index 09070d69..9546db9e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,6 @@ # installer #Makefile -# Docker -#Dockerfile - -# custom configs -custom.cfg -Makefile.config - # Python / Extensions etc. *~ *.mo @@ -27,6 +20,7 @@ __pycache__ .tox nosetests.xml unit_tests/testdata.json +build # R *.Rhistory @@ -45,20 +39,6 @@ unit_tests/testdata.json # Sublime Text Editor *.sublime* -# buildout -bin -develop-eggs -eggs -parts -build -dist -downloads -.installed.cfg -.mr.developer.cfg -bootstrap-buildout.py -bootstrap.py -#generated by buildout - # sphinx #docs/Makefile docs/make.bat diff --git a/.travis.yml b/.travis.yml index 70f46879..03a79e27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,31 @@ os: python: - "3.7" sudo: false +services: + - mongodb install: - - make install + # Python 3.x is default + - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + # Useful for debugging any issues with conda + - conda info -a + # Prepare env with Python version + - conda create -n twitcher python=$TRAVIS_PYTHON_VERSION + # Update now the env with our environment + - conda env update -f environment.yml + - source activate twitcher + # Packages for testing + - pip install -r requirements_dev.txt + # Install twitcher + - python setup.py install +before_script: + # Start WPS service on port 6000 + - pserve development.ini & + - sleep 2 script: - - make test - - make pep8 + - pytest -v -m 'not online' tests/ + - flake8 diff --git a/Makefile b/Makefile index 08d3e5db..fa3fbb03 100644 --- a/Makefile +++ b/Makefile @@ -1,46 +1,14 @@ -VERSION := 0.4.0 - -# Include custom config if it is available --include Makefile.config - # Application APP_ROOT := $(CURDIR) -APP_NAME := $(shell basename $(APP_ROOT)) - -# guess OS (Linux, Darwin, ...) -OS_NAME := $(shell uname -s 2>/dev/null || echo "unknown") -CPU_ARCH := $(shell uname -m 2>/dev/null || uname -p 2>/dev/null || echo "unknown") - -# Python -SETUPTOOLS_VERSION := 41 -CONDA_VERSION := 4.5 -BUILDOUT_VERSION := 2.13.1 +APP_NAME := twitcher # Anaconda -ANACONDA_HOME ?= $(HOME)/anaconda -CONDA_ENV ?= $(APP_NAME) -CONDA_ENVS_DIR ?= $(HOME)/.conda/envs -CONDA_ENV_PATH := $(CONDA_ENVS_DIR)/$(CONDA_ENV) -CONDA_PINNED := $(APP_ROOT)/requirements/conda_pinned +CONDA := $(shell command -v conda 2> /dev/null) +ANACONDA_HOME := $(shell conda info --base 2> /dev/null) +CONDA_ENV := $(APP_NAME) -# Configuration used by update-config -HOSTNAME ?= localhost -HTTP_PORT ?= 8094 -OUTPUT_PORT ?= 8090 - -# choose anaconda installer depending on your OS -ANACONDA_URL = https://repo.continuum.io/miniconda -ifeq "$(OS_NAME)" "Linux" -FN := Miniconda3-latest-Linux-x86_64.sh -else ifeq "$(OS_NAME)" "Darwin" -FN := Miniconda3-latest-MacOSX-x86_64.sh -else -FN := unknown -endif - -# Buildout files and folders -DOWNLOAD_CACHE := $(APP_ROOT)/downloads -BUILDOUT_FILES := parts eggs develop-eggs bin .installed.cfg .mr.developer.cfg *.egg-info bootstrap-buildout.py *.bak.* $(DOWNLOAD_CACHE) +# Temp files +TEMP_FILES := *.egg-info *.log *.sqlite # end of configuration @@ -53,11 +21,8 @@ all: help help: @echo "Please use \`make ' where is one of" @echo " help to print this help message. (Default)" - @echo " version to print version number of this Makefile." - @echo " info to print information about $(APP_NAME)." - @echo " install to install $(APP_NAME) by running 'bin/buildout -c custom.cfg'." - @echo " sysinstall to install system packages from requirements.sh. You can also call 'bash requirements.sh' directly." - @echo " update to update your application by running 'bin/buildout -o -c custom.cfg' (buildout offline mode)." + @echo " install to install $(APP_NAME) by running 'python setup.py develop'." + @echo " start to start $(APP_NAME) service as daemon (background process)." @echo " clean to delete all files that are created by running buildout." @echo "\nTesting targets:" @echo " test to run tests (but skip long running tests)." @@ -65,138 +30,60 @@ help: @echo " pep8 to run pep8 code style checks." @echo "\nSphinx targets:" @echo " docs to generate HTML documentation with Sphinx." - @echo " linkcheck to check all external links in documentation for integrity." - @echo " doc8 to run doc8 documentation style checks." - @echo "\nSupporting targets:" - @echo " envclean to remove the conda enviroment $(CONDA_ENV)." - @echo " srcclean to remove all *.pyc files." - @echo " distclean to remove *all* files that are not controlled by 'git'. WARNING: use it *only* if you know what you do!" - @echo " passwd to generate password for 'phoenix-password' in custom.cfg." - @echo " export to export the conda environment. Caution! You always need to check it the enviroment.yml is working." - @echo "\nSupervisor targets:" - @echo " start to start supervisor service." - @echo " stop to stop supervisor service." - @echo " restart to restart supervisor service." - @echo " status to show supervisor status" - -.PHONY: version -version: - @echo "Version: $(VERSION)" - -.PHONY: info -info: - @echo "Informations about your Bird:" - @echo " OS_NAME $(OS_NAME)" - @echo " CPU_ARCH $(CPU_ARCH)" - @echo " Anaconda Home $(ANACONDA_HOME)" - @echo " Conda Environment $(CONDA_ENV). Use \`source activate $(CONDA_ENV)' to activate it." - @echo " Conda Prefix $(CONDA_ENV_PATH)" - @echo " APP_NAME $(APP_NAME)" - @echo " APP_ROOT $(APP_ROOT)" - @echo " DOWNLOAD_CACHE $(DOWNLOAD_CACHE)" + @echo "\nDeployment targets:" + @echo " spec to generate Conda spec file." -## Helper targets ... ensure that Makefile etc are in place +## Conda targets -.PHONY: backup -backup: - @echo "Backup custom config ..." - @-test -f custom.cfg && cp -v --update --backup=numbered --suffix=.bak custom.cfg custom.cfg.bak - -custom.cfg: - @echo "Using custom.cfg for buildout ..." - @test -f custom.cfg || cp -v custom.cfg.example custom.cfg - -.PHONY: downloads -downloads: - @echo "Using DOWNLOAD_CACHE $(DOWNLOAD_CACHE)" - @test -d $(DOWNLOAD_CACHE) || mkdir -v -p $(DOWNLOAD_CACHE) - -.PHONY: init -init: custom.cfg downloads - -bootstrap-buildout.py: - @echo "Update buildout bootstrap-buildout.py ..." - @test -f boostrap-buildout.py || curl https://bootstrap.pypa.io/bootstrap-buildout.py --insecure --silent --output bootstrap-buildout.py - -## Anaconda targets - -.PHONY: anaconda -anaconda: - @echo "Installing Anaconda ..." - @test -d $(ANACONDA_HOME) || curl $(ANACONDA_URL)/$(FN) --silent --insecure --output "$(DOWNLOAD_CACHE)/$(FN)" - @test -d $(ANACONDA_HOME) || bash "$(DOWNLOAD_CACHE)/$(FN)" -b -p $(ANACONDA_HOME) - @echo "Add '$(ANACONDA_HOME)/bin' to your PATH variable in '.bashrc'." - -.PHONY: conda_config -conda_config: anaconda - @echo "Update ~/.condarc" - @-"$(ANACONDA_HOME)/bin/conda" install -y conda=$(CONDA_VERSION) requests - @"$(ANACONDA_HOME)/bin/conda" config --add envs_dirs $(CONDA_ENVS_DIR) - @"$(ANACONDA_HOME)/bin/conda" config --set ssl_verify true - @"$(ANACONDA_HOME)/bin/conda" config --set use_pip true - @"$(ANACONDA_HOME)/bin/conda" config --set channel_priority true - @"$(ANACONDA_HOME)/bin/conda" config --set auto_update_conda false - @"$(ANACONDA_HOME)/bin/conda" config --add channels defaults - @"$(ANACONDA_HOME)/bin/conda" config --append channels birdhouse - @"$(ANACONDA_HOME)/bin/conda" config --append channels conda-forge +.PHONY: check_conda +check_conda: +ifndef CONDA + $(error "Conda is not available. Please install miniconda: https://conda.io/miniconda.html") +endif .PHONY: conda_env -conda_env: anaconda conda_config - @echo "Update conda environment $(CONDA_ENV) ..." - @test -d $(CONDA_ENV_PATH) || "$(ANACONDA_HOME)/bin/conda" env create -n $(CONDA_ENV) -f environment.yml - "$(ANACONDA_HOME)/bin/conda" install -y -n $(CONDA_ENV) setuptools=$(SETUPTOOLS_VERSION) +conda_env: check_conda + @echo "Updating conda environment $(CONDA_ENV) ..." + "$(CONDA)" env update -n $(CONDA_ENV) -f environment.yml -.PHONY: conda_pinned -conda_pinned: conda_env - @echo "Update pinned conda packages ..." - @-test -d $(CONDA_ENV_PATH) && test -f $(CONDA_PINNED) && cp -f "$(CONDA_PINNED)" "$(CONDA_ENV_PATH)/conda-meta/pinned" +.PHONY: envclean +envclean: check_conda + @echo "Removing conda env $(CONDA_ENV)" + @-"$(CONDA)" remove -n $(CONDA_ENV) --yes --all -.PHONY: export -export: - @echo "Exporting conda enviroment ..." - @test -d $(CONDA_ENV_PATH) && "$(ANACONDA_HOME)/bin/conda" env export -n $(CONDA_ENV) -f environment.yml +.PHONY: spec +spec: check_conda + @echo "Updating conda environment specification file ..." + @-"$(CONDA)" list -n $(CONDA_ENV) --explicit > spec-file.txt ## Build targets .PHONY: bootstrap -bootstrap: init conda_env conda_pinned bootstrap-buildout.py - @echo "Bootstrap buildout ..." - @test -f bin/buildout || bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);python bootstrap-buildout.py -c custom.cfg --allow-site-packages --setuptools-version=$(SETUPTOOLS_VERSION) --buildout-version=$(BUILDOUT_VERSION)" +bootstrap: check_conda conda_env bootstrap_dev + @echo "Bootstrap ..." -.PHONY: sysinstall -sysinstall: - @echo "\nInstalling system packages for bootstrap ..." - @bash bootstrap.sh -i - @echo "\nInstalling system packages for your application ..." - @-test -f requirements.sh && bash requirements.sh +.PHONY: bootstrap_dev +bootstrap_dev: + @echo "Installing development requirements for tests and docs ..." + @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && pip install -r requirements_dev.txt" .PHONY: install install: bootstrap - @echo "Installing application with buildout ..." - @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);bin/buildout buildout:anaconda-home=$(ANACONDA_HOME) -c custom.cfg" + @echo "Installing application ..." + @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && python setup.py develop" @echo "\nStart service with \`make start'" -.PHONY: update -update: - @echo "Update application config with buildout (offline mode) ..." - @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);bin/buildout buildout:anaconda-home=$(ANACONDA_HOME) -o -c custom.cfg" - -.PHONY: update-config -update-config: - @echo "Update application config with buildout (offline mode) and environment variables..." - @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);bin/buildout buildout:anaconda-home=$(ANACONDA_HOME) settings:hostname=$(HOSTNAME) settings:output-port=$(OUTPUT_PORT) settings:http-port=$(HTTP_PORT) -o -c custom.cfg" +.PHONY: start +start: check_conda + @echo "Starting application ..." + @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && pserve development.ini &" .PHONY: clean clean: srcclean envclean - @echo "Cleaning buildout files ..." - @-for i in $(BUILDOUT_FILES); do \ - test -e $$i && rm -v -rf $$i; \ - done - -.PHONY: envclean -envclean: stop - @echo "Removing conda env $(CONDA_ENV)" - @-test -d $(CONDA_ENV_PATH) && "$(ANACONDA_HOME)/bin/conda" remove -n $(CONDA_ENV) --yes --all + @echo "Cleaning generated files ..." + @-for i in $(TEMP_FILES); do \ + test -e $$i && rm -v -rf $$i; \ + done .PHONY: srcclean srcclean: @@ -204,68 +91,32 @@ srcclean: @-find $(APP_ROOT) -type f -name "*.pyc" -print | xargs rm .PHONY: distclean -distclean: backup clean - @echo "Cleaning distribution ..." +distclean: clean + @echo "Cleaning ..." @git diff --quiet HEAD || echo "There are uncommited changes! Not doing 'git clean' ..." - @-git clean -dfx -e *.bak -e custom.cfg -e Makefile.config + @-git clean -dfx -.PHONY: passwd -passwd: custom.cfg - @echo "Generate Phoenix password ..." - @echo "Enter a password with at least 8 characters." - @bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV); python -c 'from IPython.lib import passwd; pw = passwd(algorithm=\"sha256\"); lines = [\"phoenix-password = \" + pw + \"\\n\" if line.startswith(\"phoenix-password\") else line for line in open(\"custom.cfg\", \"r\")]; file = open(\"custom.cfg\", \"w\"); file.writelines(lines); file.close()'" - @echo "" - @echo "Run \`make install restart' to activate this password." +## Test targets .PHONY: test -test: +test: check_conda @echo "Running tests (skip slow and online tests) ..." - bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV); bin/py.test -v -m 'not slow and not online'" + @bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);pytest -v -m 'not slow and not online' tests/" .PHONY: testall -testall: +testall: check_conda @echo "Running all tests (including slow and online tests) ..." - bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV); bin/py.test -v" + @bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && pytest -v tests/" .PHONY: pep8 -pep8: +pep8: check_conda @echo "Running pep8 code style checks ..." - $(CONDA_ENV_PATH)/bin/flake8 + @bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && flake8" + +## Sphinx targets .PHONY: docs -docs: +docs: check_conda @echo "Generating docs with Sphinx ..." - $(MAKE) -C $@ clean html - @echo "open your browser: firefox docs/build/html/index.html" - -.PHONY: linkcheck -linkcheck: - @echo "Run link checker on docs..." - $(MAKE) -C docs linkcheck - -.PHONY: doc8 -doc8: - @echo "Running doc8 doc style checks ..." - $(CONDA_ENV_PATH)/bin/doc8 docs/ - -## Supervisor targets - -.PHONY: start -start: - @echo "Starting supervisor service ..." - bin/supervisord start - -.PHONY: stop -stop: - @echo "Stopping supervisor service ..." - -bin/supervisord stop - -.PHONY: restart -restart: - @echo "Restarting supervisor service ..." - bin/supervisord restart - -.PHONY: status -status: - @echo "Supervisor status ..." - bin/supervisorctl status + @-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);$(MAKE) -C $@ clean html" + @echo "open your browser: open docs/build/html/index.html" diff --git a/Makefile.config.example b/Makefile.config.example deleted file mode 100644 index fccdcaa5..00000000 --- a/Makefile.config.example +++ /dev/null @@ -1,3 +0,0 @@ -# Anaconda -ANACONDA_HOME ?= /opt/anaconda -CONDA_ENVS_DIR ?= /opt/anaconda/envs diff --git a/README.rst b/README.rst index b8bfb37e..51f726f0 100644 --- a/README.rst +++ b/README.rst @@ -35,7 +35,7 @@ Twitcher extensions: * `Magpie`_ is an AuthN/AuthZ service provided by the `PAVICS`_ project. -Twitcher is a **prototype** implemented in Python with the `Pyramid`_ web framework. +Twitcher is implemented with the Python `Pyramid`_ web framework. Twitcher is part of the `Birdhouse`_ project. The documentation is on `ReadTheDocs`_. diff --git a/bootstrap.sh b/bootstrap.sh deleted file mode 100755 index 8f4dc1c0..00000000 --- a/bootstrap.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -usage() { - cat </dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext - +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " applehelp to make an Apple Help Book" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - @echo " coverage to run coverage check of the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/twitcher.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/twitcher.qhc" - -applehelp: - $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp - @echo - @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." - @echo "N.B. You won't be able to view it unless you put it in" "~/Library/Documentation/Help or install it in your application" "bundle." - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/twitcher" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/twitcher" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " "results in $(BUILDDIR)/doctest/output.txt." - -coverage: - $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage - @echo "Testing of coverage in the sources finished, look at the " "results in $(BUILDDIR)/coverage/python.txt." + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." +.PHONY: help Makefile -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source/_static/README.txt b/docs/source/_static/README.txt new file mode 100644 index 00000000..764fc26e --- /dev/null +++ b/docs/source/_static/README.txt @@ -0,0 +1 @@ +Folder for static files diff --git a/docs/source/_static/birdhouse_logo.svg b/docs/source/_static/birdhouse_logo.svg new file mode 100644 index 00000000..3d74cc46 --- /dev/null +++ b/docs/source/_static/birdhouse_logo.svg @@ -0,0 +1,79 @@ + + + + + + + + + + + diff --git a/docs/source/_static/favicon.ico b/docs/source/_static/favicon.ico new file mode 100644 index 00000000..51386f61 Binary files /dev/null and b/docs/source/_static/favicon.ico differ diff --git a/docs/source/api.rst b/docs/source/api.rst index 1e81474c..2ff175db 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -1,8 +1,6 @@ .. toctree:: :hidden: - autoapi/twitcher - .. _api: ************************* @@ -19,7 +17,7 @@ To use the XML-RPC interface, connect to twitcher’s HTTPS port with any XML-RP .. code-block:: python import xmlrpclib - server = xmlrpclib.Server('https://localhost:8000/RPC2') + server = xmlrpclib.Server('http://localhost:8000/RPC2') .. warning:: @@ -30,6 +28,7 @@ To use the XML-RPC interface, connect to twitcher’s HTTPS port with any XML-RP The `XML-RPC `_ interface can also be accessed from Java and other languages. -See the ``twitcher/rpcinterface.py`` module for the available xmlrpc methods. +See the ``twitcher/rpcinterface.py`` module for the available xmlrpc methods: -:doc:`Internal Documentation Reference. ` +.. autoclass:: twitcher.rpcinterface.RPCInterface + :members: diff --git a/docs/source/appendix.rst b/docs/source/appendix.rst deleted file mode 100644 index afa7e1b6..00000000 --- a/docs/source/appendix.rst +++ /dev/null @@ -1,69 +0,0 @@ -.. _appendix: - -************ -Useful Links -************ - -Twitcher alike projects: - -* `Climate4Impact OWS token api `_ - -OWS Proxies: - -* `owsproxy by mapbender `_ -* http://doc.mapbender3.org/en/book/development/proxy.html -* https://github.com/camptocamp/secureOWS -* https://github.com/elemoine/papyrus_ogcproxy -* http://proxy4ows.org/ -* http://www.slideshare.net/jachym/proxy4ows -* http://wiki.deegree.org/deegreeWiki/deegree3/SecurityRequirements - -Other Proxies: - -* https://mapproxy.org/ - - -Security Filters: - -* `FOSS4G Talk `_ on using `mod_python `_ with security filter for PyWPS. -* GeoFence: https://github.com/georchestra/geofence -* georchestra: http://www.georchestra.org/ (security proxy for inspire) -* Geoserver Security: http://docs.geoserver.org/stable/en/user/security/service.html -* WSGI middleware for oauth: https://pypi.python.org/pypi/wsgioauth -* https://pypi.python.org/pypi/stups-tokens -* http://docs.services.mozilla.com/token/index.html -* https://github.com/mozilla-services/tokenserver -* https://www.mapbox.com/developers/api/ -* https://pypi.python.org/pypi/django-access-tokens -* https://github.com/jpulgarin/django-tokenapi -* https://github.com/repoze/repoze.who - -Pyramid: - -* https://github.com/elemoine/papyrus -* https://github.com/elemoine/papyrus_mapproxy -* `http://pythonpaste.org/modules/proxy.html` - -Macaroons Tokens (simple security tokens for distributed systems): - -* https://github.com/rescrv/libmacaroons -* http://hackingdistributed.com/2014/05/16/macaroons-are-better-than-cookies/ -* https://github.com/shirkey/macaroons-kopdar/ - -JSON Web Token (JWT): - -* http://openid.net/specs/draft-jones-oauth-jwt-bearer-03.html -* https://pypi.python.org/pypi/pyramid_jwtauth -* https://pypi.python.org/pypi/pyramid_jwt - -Simple Web Token (SWT): - -* https://lbadri.wordpress.com/2012/07/30/anatomy-of-a-simple-web-token-swt/ -* https://lbadri.wordpress.com/2012/09/13/simple-web-token-swt-as-oauth-2-0-bearer-token-for-asp-net-web-api/ -* https://msdn.microsoft.com/en-us/library/hh454950.aspx -* https://en.wikipedia.org/wiki/Access_Control_Service - -Geo XACML (access control XML): - -* https://en.wikipedia.org/wiki/XACML -* https://en.wikipedia.org/wiki/GeoXACML diff --git a/docs/source/conf.py b/docs/source/conf.py index c1e15c44..b73b8a97 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -11,36 +11,30 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os -import shlex +import sys +sys.path.insert(0, os.path.abspath('../../')) # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.6' +needs_sphinx = '1.7' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'autoapi.extension', - 'sphinx.ext.intersphinx', + 'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', ] -autoapi_type = 'python' -autoapi_dirs = ['../../twitcher'] -autoapi_file_pattern = '*.py' -autoapi_options = ['members', 'undoc-members', 'private-members'] - # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -50,14 +44,14 @@ source_suffix = '.rst' # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'twitcher' -copyright = u'2018, Birdhouse' +copyright = u'2019, Birdhouse' author = u'Birdhouse' # The version info for the project you're documenting, acts as replacement for @@ -65,9 +59,9 @@ # built documents. # # The short X.Y version. -version = '0.4' +version = '' # The full version, including alpha/beta/rc tags. -release = '0.4' +release = '0.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -139,12 +133,12 @@ # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +html_logo = "_static/birdhouse_logo.svg" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +html_favicon = "_static/favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -294,43 +288,3 @@ # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False - - -# Example configuration for intersphinx: refer to the Python standard library. -#intersphinx_mapping = {'https://docs.python.org/': None} -intersphinx_mapping = {'python': ('http://docs.python.org/', None), - 'birdhouse': ('http://birdhouse.readthedocs.io/en/latest/', None), - 'phoenix': ('http://pyramid-phoenix.readthedocs.io/en/latest/', None), - 'malleefowl': ('http://malleefowl.readthedocs.io/en/latest/', None), - 'twitcher': ('http://twitcher.readthedocs.io/en/latest/', None), - 'flyingpigeon': ('http://flyingpigeon.readthedocs.io/en/latest/', None), - 'hummingbird': ('http://birdhouse-hummingbird.readthedocs.io/en/latest/', None), - 'emu': ('http://emu.readthedocs.io/en/latest/', None), - 'birdy': ('http://birdy.readthedocs.io/en/latest/', None), - 'bootstrap': ('http://birdhousebuilderbootstrap.readthedocs.io/en/latest/', None), - } - -# linkcheck options -# http://www.sphinx-doc.org/en/stable/config.html?highlight=linkchecker#options-for-the-linkcheck-builder -linkcheck_ignore = [r'http[s]*://localhost.*/', 'https://mouflon.dkrz.de/', 'https://esgf-data.dkrz.de/', 'https://indico.egi.eu/'] -linkcheck_timeout = 15 - -# Link references always present on RST page. -rst_epilog = """ -.. _Sphinx: http://sphinx-doc.org/ -.. _reStructuredText: http://sphinx-doc.org/rest.html -.. _Read the Docs: https://readthedocs.org -.. _Anaconda: https://www.continuum.io/ -.. _Buildout: http://www.buildout.org/en/latest/ -.. _Phoenix: http://pyramid-phoenix.readthedocs.io/en/latest/ -.. _Malleefowl: http://malleefowl.readthedocs.io/en/latest/ -.. _Twitcher: http://twitcher.readthedocs.io/en/latest/ -.. _Flyingpigeon: http://flyingpigeon.readthedocs.io/en/latest/ -.. _Hummingbird: http://birdhouse-hummingbird.readthedocs.io/en/latest/ -.. _Emu: http://emu.readthedocs.io/en/latest/ -.. _Birdy: http://birdy.readthedocs.io/en/latest/ -.. _Bootstrap: http://birdhousebuilderbootstrap.readthedocs.io/en/latest/ -.. _icclim: http://icclim.readthedocs.io/en/latest/ -.. _PyWPS: http://pywps.org/ -.. _dispel4py: https://github.com/dispel4py/dispel4py -""" diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index de2ac2ab..a7f13185 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -1,67 +1,7 @@ .. _configuration: -****************** -Configuration File -****************** +Configuration +============= -.. contents:: - :local: - :depth: 2 - -After you have installed twitcher you can customize the default twitcher configuration by editing the ``custom.cfg`` configuration file. This configuration file overwrites the default settings in the ``buildout.cfg``: - -.. code-block:: cfg - - $ vim custom.cfg - $ cat custom.cfg - [buildout] - extends = buildout.cfg - - [settings] - hostname = localhost - https-port = 8000 - log-level = WARN - username = - password = - workdir = - ows-security = true - ows-proxy = true - rpcinterface = true - -After your have made a change in ``custom.cfg`` you *need to update* the installation and restart the twitcher service: - -.. code-block:: sh - - $ make update - $ make restart - $ make status - -Set hostname and port -===================== - -Edit the options ``hostname`` and ``https-port``. - - -Activate basic-auth for XML-RPC control interface -================================================= - -Set ``username`` and ``password``. - -Deactivate twitcher components -============================== - -Twitcher has four components which by default are activated: - -ows-security - The OWS security wsgi middleware -ows-proxy - A proxy wsgi application for OWS services -rpcinterface - An XML-RPC interface to control token generation and service registration - -By setting a component option to ``false`` you can deactivate it: - -.. code-block:: sh - - [settings] - ows-proxy = false +Twitcher has a configuration file `development.ini` for local development. +Copy and edit this configuration to adapt to your settings. diff --git a/docs/source/dev_guide.rst b/docs/source/dev_guide.rst new file mode 100644 index 00000000..d6901f12 --- /dev/null +++ b/docs/source/dev_guide.rst @@ -0,0 +1,95 @@ +.. _devguide: + +Developer Guide +=============== + +.. contents:: + :local: + :depth: 1 + +Building the docs +----------------- + +First install dependencies for the documentation: + +.. code-block:: console + + $ make bootstrap_dev + $ make docs + +.. _testing: + +Running tests +------------- + +Run tests using `pytest`_. + +First activate the ``twitcher`` Conda environment and install ``pytest``. + +.. code-block:: console + + $ source activate twitcher + $ pip install -r requirements_dev.txt # if not already installed + +Run quick tests (skip slow and online): + +.. code-block:: console + + $ pytest -m 'not slow and not online'" + +Run all tests: + +.. code-block:: console + + $ pytest + +Check pep8: + +.. code-block:: console + + $ flake8 + +Run tests the lazy way +---------------------- + +Do the same as above using the ``Makefile``. + +.. code-block:: console + + $ make test + $ make testall + $ make pep8 + +Prepare a release +----------------- + +Update the Conda specification file to build identical environments_ on a specific OS. + +.. note:: You should run this on your target OS, in our case Linux. + +.. code-block:: console + + $ make clean + $ make install + $ make spec + +.. _`environments`: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#building-identical-conda-environments + + +Bump a new version +------------------ + +Make a new version of twitcher in the following steps: + +* Make sure everything is commit to GitHub. +* Update ``CHANGES.rst`` with the next version. +* Dry Run: ``bumpversion --dry-run --verbose --new-version 0.5.1 patch`` +* Do it: ``bumpversion --new-version 0.5.1 patch`` +* ... or: ``bumpversion --new-version 0.6.0 minor`` +* Push it: ``git push`` +* Push tag: ``git push --tags`` + +See the bumpversion_ documentation for details. + +.. _bumpversion: https://pypi.org/project/bumpversion/ +.. _pytest: https://docs.pytest.org/en/latest/ diff --git a/docs/source/index.rst b/docs/source/index.rst index 0b60bf65..c9fe7130 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,8 +6,8 @@ overview installation configuration + dev_guide running tutorial api changes - appendix diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 44c3b482..269ce939 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -4,47 +4,59 @@ Installation ************ -The installation is using the Python distribution system `Anaconda`_ to maintain software dependencies. `Buildout`_ is used to setup the application with all services and configuration files. +From GitHub Sources +=================== -Requirements -============ +Install twitcher from GitHub sources: -The installation works on Linux 64 bit distributions (tested on Ubuntu 14.04) and also on MacOS (tested on Sierra). +.. code-block:: console -From GitHub Sources -=================== + $ git clone https://github.com/bird-house/twitcher.git + $ cd twitcher + $ conda env update -f environment.yml + $ conda activate twitcher + $ python setup.py install + +The installation process setups a Conda_ environment named *twitcher* +with all dependent packages. -Install twitcher as normal user from GitHub sources: +... or do it the lazy way ++++++++++++++++++++++++++ + +We provide also a ``Makefile`` to run this installation: .. code-block:: sh $ git clone https://github.com/bird-house/twitcher.git $ cd twitcher - $ make clean install - $ make test + $ make clean # cleans up a previous Conda environment + $ make install # runs the above installation steps -The installation process setups a conda environment named *twitcher* with all dependent conda (and pip) packages. The installation folder (for configuration files, database etc) is by default ``~/birdhouse``. Configuration options can be overriden in the buildout ``custom.cfg`` file. +Starting Database +================= -Starting Twitcher Service -========================= +Twitcher is using a MongoDB_ database. +The default configuration is using the MongoDB standard port 27017 on localhost. -Twitcher is run as `Gunicorn `_ WSGI application server behind the `Nginx `_ HTTP server. Starting/Stopping the services is controlled by `Supervisor `_. This is described in the `Birdhouse documenation `_. +You can install MongoDB with Conda_ for testing: -Start the twitcher service (using supervisor): +.. code-block:: console -.. code-block:: sh + $ conda install mongodb=4 + # start DB for testing + $ mongod --config etc/mongod.conf - $ make start # or make restart +Starting Twitcher Service +========================= -Check the status of the twitcher service: +For development twitcher is using the the waitress_ WSGI server. -.. code-block:: sh +Start the twitcher service using the `development.ini` configuration: - $ make status - Supervisor status ... - mongodb RUNNING pid 6863, uptime 0:00:19 - nginx RUNNING pid 6865, uptime 0:00:19 - twitcher RUNNING pid 6864, uptime 0:00:19 +.. code-block:: console + $ pserve development.ini --reload -You will find more information about the installation in the `Makefile documentation `_. +.. _waitress: https://docs.pylonsproject.org/projects/waitress/en/latest/ +.. _Conda: https://conda.io/en/latest/ +.. _MongoDB: https://www.mongodb.com/ diff --git a/docs/source/overview.rst b/docs/source/overview.rst index 2b3e8da0..e5755c04 100644 --- a/docs/source/overview.rst +++ b/docs/source/overview.rst @@ -19,7 +19,8 @@ The following image gives an overview of the Twitcher components. The Aim ======= -The aim is to have a simple to use security filter for OGC/OWS services (especially for Web Processing Services) which can be integrated in existing processing infrastructures. +The aim is to have a simple to use security filter for OGC/OWS services (especially for Web Processing Services) +which can be integrated in existing processing infrastructures. The design aims are: @@ -27,16 +28,9 @@ The design aims are: * Simple: to use, to deploy, to maintain, to understand, ... * Token based security. * Tokens are only valid for a short period of time. -* Able to provide additional data (user/process environment) with a security token. * Able to be used in a distributed infrastructure (maybe using `Macaroons `_?). * Python .. *ehm* not a requirement ... but more and more used. -Why Twitcher? -============= - -Unfortunately we haven't found anything that fulfills our requirements. There are some other projects with the same subject and give inspiration but they don't work for us. Please see :ref:`appendix`. - - Twitcher Components =================== @@ -45,21 +39,25 @@ Twitcher consists of the following main parts: OWS Security A wsgi middleware (actually currently it is a `Pyramid tween `_) which puts a simple token based security layer on top of a wsgi application. The access tokens are stored in a MongoDB. OWS Proxy - A wsgi application which acts as a proxy for registred OWS services. Currently it only supports WPS services. + A wsgi middleware which acts as a proxy for registred OWS services. + Currently it only supports WPS services. XML-RPC Interface - An XML-RPC service which is used to control the token generation and OWS service registration. The interface is accessed using Basic Authentication. It should be used by an administrator and administrative web portals. + An XML-RPC service which is used to control the token generation and OWS service registration. + The interface is accessed using Basic Authentication. It should be used by an administrator and administrative web portals. How to Use ========== The OWS security middleware protects OWS services with a simple string based token mechanism. A WPS client needs to provide a string token to access the internal WPS or a registered OWS service. -A token is generated via the XML-RPC interface. This interface is supposed to be used by an external administration client which has user authentication and generates an access token on behalf of the user. - -The admin interface can add additional data (environment variables) to the access token. This additional data is stored with the token in the token store. The data will be added to the *wsgi environ* of the WPS application when the token is provided by the client request.This data can be used to set a dynamic configuration of the WPS service, e.a. user specific data access credentials which are used by the WPS processes. External WPS/OWS services, which are registered with the OWS proxy, have no access to this additional data. +A token is generated via the XML-RPC interface. This interface is supposed to be used by +an external administration client which has user authentication and generates an access token on behalf of the user. -The OWS security middleware works currently only with WPS services. It allows by default to use ``GetCapabilities`` and ``DescribeProcess`` requests without a token. The ``Execute`` request (and anything else) can be accessed only with a valid token. +The OWS security middleware works currently only with WPS services. +It allows by default to use ``GetCapabilities`` and ``DescribeProcess`` requests without a token. +The ``Execute`` request (and anything else) can be accessed only with a valid token. -Access tokens have a life time limit. By default only for one hour but the admin can set a longer life time when generating a token. +Access tokens have a life time limit. By default only for one hour but the admin +can set a longer life time when generating a token. -So, twitcher is meant to be integrated in existing processing infrastructures with OGC/OWS services and portals. You can use twitcher as a standalone service but currently only for development and demo purposes. +Twitcher is meant to be integrated in existing processing infrastructures with OGC/OWS services. diff --git a/docs/source/running.rst b/docs/source/running.rst index 7863a1ce..71d54d74 100644 --- a/docs/source/running.rst +++ b/docs/source/running.rst @@ -9,35 +9,23 @@ Running Twitcher :depth: 2 -Running twitcher service -======================== - -The twitcher service is controlled by `supervisor `_. The twitcher installation comes with a Makefile which provides shortcut commands for supervisor: - -.. code-block:: sh - - $ cd twitcher # cd into twitcher installation directory - $ make status # show running supervisor services (incl. twitcher) - $ make start # start all supervisor services (incl. twitcher) - $ make stop # stop ... - $ make restart # restart ... - Running `twitcherctl` ===================== -The ``twitcherctl`` is a command line tool to control the twitcher service. It uses the XML-RPC api of twitcher to generate access tokens and to register OWS services. +The ``twitcherctl`` is a command line tool to control the twitcher service. +It uses the XML-RPC api of twitcher to generate access tokens and to register OWS services. -``twitcherctl`` is part of the twitcher installation. When you have installed twitcher from GitHub then start ``twitcherctl`` with: +``twitcherctl`` is part of the twitcher installation. +When you have installed twitcher from GitHub then start ``twitcherctl`` with: -.. code-block:: sh +.. code-block:: console - $ cd twitcher # cd into twitcher installation directory - $ bin/twitcherctl -h + $ twitcherctl -h `twitcherctl` Commands and Options ------------------------------------------- +---------------------------------- ``twitcherctl`` has the following command line options: @@ -82,15 +70,15 @@ Generate an access token See the available options: -.. code-block:: sh +.. code-block:: console - $ bin/twitcherctl -k gentoken -h + $ twitcherctl -k gentoken -h Generate an access token valid for 24 hours (use ``-k`` to avoid validation of HTTPS server certificate): -.. code-block:: sh +.. code-block:: console - $ bin/twitcherctl -k gentoken -H 24 + $ twitcherctl -k gentoken -H 24 Register an OWS Service for the OWS Proxy @@ -98,18 +86,19 @@ Register an OWS Service for the OWS Proxy See the available options: -.. code-block:: sh +.. code-block:: console - bin/twitcherctl -k register -h + twitcherctl -k register -h Register a local WPS service: -.. code-block:: sh +.. code-block:: console - $ bin/twitcherctl -k register http://localhost:5000/wps + $ twitcherctl -k register http://localhost:5000/wps tiny_buzzard -You can use the ``--name`` option to provide a name (used by the OWS proxy). Otherwise a nice name will be generated. +You can use the ``--name`` option to provide a name (used by the OWS proxy). +Otherwise a nice name will be generated. Show Status of Twitcher @@ -117,26 +106,17 @@ Show Status of Twitcher Currently the ``status`` command shows only the registered OWS services: -.. code-block:: sh +.. code-block:: console - $ bin/twitcherctl -k list + $ twitcherctl -k list [{'url': 'http://localhost:5000/wps', 'proxy_url': 'https://localhost:38083/ows/proxy/tiny_buzzard', 'type': 'wps', 'name': 'tiny_buzzard'}] -Using OWSProxy -============== - -See the :ref:`tutorial`. - - -Using WPS Application -===================== - -See the :ref:`tutorial`. Use Twitcher components in your Pyramid Application =================================================== -Instead of running twitcher as a service you can also include twitcher components (OWS Security Middleware, OWS Proxy) in a Pyramid application. +Instead of running twitcher as a service you can also include twitcher components +(OWS Security Middleware, OWS Proxy) in a Pyramid application. Include OWS Security Middleware ------------------------------- diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 14f3690e..dc2cc562 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -17,7 +17,7 @@ The ``OWSProxy`` is a proxy service for OWS services. Currently it only supports First you need an external WPS. You can use `Emu WPS service `_ from Birdhouse. Get it from GitHub and run the installation: -.. code-block:: sh +.. code-block:: console $ git clone https://github.com/bird-house/emu.git $ cd emu @@ -30,28 +30,27 @@ http://localhost:5000/wps?service=WPS&version=1.0.0&request=GetCapabilities Make sure Twitcher is installed and running: -.. code-block:: sh +.. code-block:: console $ cd ../twitcher # cd into the twitcher installation folder - $ make restart - $ make status + $ pserve development.ini Register a WPS service ---------------------- Register the Emu WPS service at the Twitcher ``OWSProxy``: -.. code-block:: sh +.. code-block:: console - $ bin/twitcherctl -k register --name emu http://localhost:5000/wps + $ twitcherctl -k register --name emu http://localhost:5000/wps If you don't provide a name with ``--name`` option then a nice name will be generated, for example ``sleepy_flamingo``. Use the ``list`` command to see which WPS services are registered with OWSProxy: -.. code-block:: sh +.. code-block:: console - $ bin/twitcherctl -k list + $ twitcherctl -k list [{'url': 'http://localhost:5000/wps', 'proxy_url': 'https://localhost:8000/ows/proxy/emu', 'type': 'wps', 'name': 'emu'}] @@ -63,16 +62,16 @@ Replace the ``service_name`` with the registered name. Run a ``GetCapabilities`` request for the registered Emu WPS service: -.. code-block:: sh +.. code-block:: console - $ curl -k "https://localhost:8000/ows/proxy/emu?service=wps&request=getcapabilities" + $ curl -k "http://localhost:8000/ows/proxy/emu?service=wps&request=getcapabilities" Run a ``DescribeProcess`` request: -.. code-block:: sh +.. code-block:: console - $ curl -k "https://localhost:8000/ows/proxy/emu?service=wps&request=describeprocess&identifier=hello&version=1.0.0" + $ curl -k "http://localhost:8000/ows/proxy/emu?service=wps&request=describeprocess&identifier=hello&version=1.0.0" Use tokens to run an execute request ------------------------------------ @@ -81,20 +80,21 @@ By default the WPS service is protected by the ``OWSSecurity`` wsgi middleware. Run an ``Exceute`` request: -.. code-block:: sh +.. code-block:: console - $ curl -k "https://localhost:8000/ows/proxy/emu?service=wps&request=execute&identifier=hello&version=1.0.0&datainputs=name=tux" + $ curl -k "http://localhost:8000/ows/proxy/emu?service=wps&request=execute&identifier=hello&version=1.0.0&datainputs=name=tux" Now you should get an XML error response with a message that you need to provide an access token (see section above). We need to generate an access token with ``twitcherctl``: -.. code-block:: sh +.. code-block:: console - $ bin/twitcherctl -k gentoken -H 24 + $ twitcherctl -k gentoken -H 24 def456 -By default the token has a limited life time of one hour. With the option ``-H`` you can extend the life time in hours (24 hours in this example). +By default the token has a limited life time of one hour. +With the option ``-H`` you can extend the life time in hours (24 hours in this example). You can provide the access token in three ways (see section above): @@ -104,9 +104,9 @@ You can provide the access token in three ways (see section above): In the following example we provide the token as HTTP parameter: -.. code-block:: sh +.. code-block:: console - $ curl -k "https://localhost:8000/ows/proxy/emu?service=wps&request=execute&identifier=hello&version=1.0.0&datainputs=name=tux&token=def456" + $ curl -k "http://localhost:8000/ows/proxy/emu?service=wps&request=execute&identifier=hello&version=1.0.0&datainputs=name=tux&token=def456" .. warning:: @@ -116,154 +116,40 @@ In the following example we provide the token as HTTP parameter: Use x509 certificates to control client access ================================================== +.. warning:: + + You need the Nginx web-service in front of the Twitcher WSGI service to use x509 certificates. + Since version 0.3.6 Twitcher is prepared to use x509 certificates to control client access. -By default it is configured to accept x509 proxy certificates from `ESGF`_. +By default it is configured to accept x509 proxy certificates from ESGF_. Register the Emu WPS service at the Twitcher ``OWSProxy`` with ``auth`` option ``cert``: -.. code-block:: sh +.. code-block:: console - $ bin/twitcherctl -k register --name emu --auth cert http://localhost:5000/wps + $ twitcherctl -k register --name emu --auth cert http://localhost:5000/wps The ``GetCapabilities`` and ``DescribeProcess`` requests are not blocked: -.. code-block:: sh +.. code-block:: console - $ curl -k "https://localhost:8000/ows/proxy/emu?service=wps&request=getcapabilities" - $ curl -k "https://localhost:8000/ows/proxy/emu?service=wps&request=describeprocess&identifier=hello&version=1.0.0" + $ curl -k "http://localhost:8000/ows/proxy/emu?service=wps&request=getcapabilities" + $ curl -k "http://localhost:8000/ows/proxy/emu?service=wps&request=describeprocess&identifier=hello&version=1.0.0" When you run an ``Exceute`` request without a certificate you should get an exception report: -.. code-block:: sh +.. code-block:: console - $ curl -k "https://localhost:8000/ows/proxy/emu?service=wps&request=execute&identifier=hello&version=1.0.0&datainputs=name=tux" + $ curl -k "http://localhost:8000/ows/proxy/emu?service=wps&request=execute&identifier=hello&version=1.0.0&datainputs=name=tux" Now you should get an XML error response with a message that you need to provide a valid X509 certificate. Get a valid proxy certificate from ESGF, you may use the `esgf-pyclient`_ to run a myproxy logon. Let's say your proxy certificate is ``cert.pem``, then run the exceute request again using this certificate: -.. code-block:: sh - - $ curl --cert cert.pem --key cert.pem -k "https://localhost:8000/ows/proxy/emu?service=wps&request=execute&identifier=hello&version=1.0.0&datainputs=name=tux" - - -Use Birdy WPS command line client to run a Process -================================================== - - -Install the `birdy `_ WPS command line client: - -.. code-block:: sh - - $ conda install -c birdhouse birdhouse-birdy - -If ``conda`` is not in your path ... it was installed by the twitcher installer and is by default in ``~/anaconda/bin``. - -Generate a new access token: - -.. code-block:: sh - - $ cd twitcher # cd into twitcher installation folder - $ bin/twitcherctl -k gentoken - 98765 - -Check which WPS is registered (or register one as described above): - -.. code-block:: sh - - $ bin/twitcherctl -k list - [{'url': 'http://localhost:5000/wps', 'proxy_url': 'https://localhost:8000/ows/proxy/emu', 'type': 'wps', 'name': 'emu'}] - - -Set the ``WPS_SERVICE`` environment variable for birdy with the ``proxy_url`` and extended with **access token**: - -.. code-block:: sh - - $ export WPS_SERVICE=https://localhost:8000/ows/proxy/emu/98765 - - -Now, run birdy: - -.. code-block:: sh - - $ birdy -h - -You get a list of available WPS processes:: - - usage: brdy [] [] - - Emu: WPS processes for testing and demos. - - optional arguments: - -h, --help show this help message and exit - --debug enable debug mode - - command: - List of available commands (wps processes) - - {helloworld,ultimatequestionprocess,dummyprocess,wordcount,inout,multiplesources,chomsky,zonal_mean} - Run "birdy -h" to get additional help. - helloworld Hello World: Welcome user and say hello ... - ultimatequestionprocess - Answer to Life, the Universe and Everything: Numerical - solution that is the answer to Life, Universe and - Everything. The process is an improvement to Deep - Tought computer (therefore version 2.0) since it no - longer takes 7.5 milion years, but only a few seconds - to give a response, with an update of status every 10 - seconds. - dummyprocess Dummy Process: The Dummy process is used for testing - the WPS structure. The process will accept 2 input - numbers and will return the XML result with an add one - and subtract one operation - wordcount Word Counter: Counts words in a given text ... - inout Testing all Data Types: Just testing data types like - date, datetime etc ... - multiplesources Multiple Sources: Process with multiple different - sources ... - chomsky Chomsky text generator: Generates a random chomsky - text ... - - -Show params of ``helloworld process``: - -.. code-block:: sh - - $ birdy helloworld -h - - -You get a list of input/output params as option:: - - usage: birdy helloworld [-h] --user [USER] - [--output [{output} [{output} ...]]] - - optional arguments: - -h, --help show this help message and exit - --user [USER] Your name: Please enter your name - --output [{output} [{output} ...]] - Output: output=Welcome message: None (default: all - outputs) - - -Run the ``helloworld`` process: - -.. code-block:: sh - - $ birdy helloworld --user pingu - -The process output:: - - INFO:Execution status: ProcessAccepted - INFO:Execution status: ProcessSucceeded - INFO:Output: - INFO:output=Hello pingu and welcome to WPS :) - - -If you don't provide a token or the token is invalid then you will get an error message:: +.. code-block:: console - owslib.wps.WPSException : {'locator': 'AccessForbidden', 'code': 'NoApplicableCode', 'text': 'Access token is required to access this service.'} - WARNING:Error: code=NoApplicableCode, locator=AccessForbidden, text=Access token is required to access this service. + $ curl --cert cert.pem --key cert.pem -k "http://localhost:8000/ows/proxy/emu?service=wps&request=execute&identifier=hello&version=1.0.0&datainputs=name=tux" .. _ESGF: https://esgf.llnl.gov/ diff --git a/environment.yml b/environment.yml index 9518c760..5f3b20e7 100644 --- a/environment.yml +++ b/environment.yml @@ -2,39 +2,8 @@ name: twitcher channels: - defaults dependencies: -- python=3.7 -- curl -- pyopenssl -- cryptography -- mako -- pyyaml -# tests -- pytest -- flake8 -- mock -- pbr -- webtest -# sphinx -- sphinx +- python>=3.7 +- pip # twitcher -- argcomplete - lxml -- libxslt -- paste -- pastedeploy - requests -- requests-oauthlib -- webob -# nginx, supervisor -- nginx -- supervisor=4 -- gunicorn=19 -- gevent -# mongodb -- pymongo=3 -- mongodb=3.4 -- pip: - - genshi==0.7 - - pyramid - - pyramid-rpc - - sphinx-autoapi diff --git a/etc/mongod.conf b/etc/mongod.conf new file mode 100644 index 00000000..dd4b2a37 --- /dev/null +++ b/etc/mongod.conf @@ -0,0 +1,8 @@ +systemLog: + destination: file + path: /usr/local/var/log/mongodb/mongo.log + logAppend: true +storage: + dbPath: /usr/local/var/mongodb +net: + bindIp: 127.0.0.1 diff --git a/templates/nginx.conf b/etc/nginx.conf similarity index 79% rename from templates/nginx.conf rename to etc/nginx.conf index e494451c..c1b41af9 100644 --- a/templates/nginx.conf +++ b/etc/nginx.conf @@ -1,23 +1,22 @@ -# Phoenix: a pyramid web frontend for WPS upstream twitcher { - server unix://${socket} fail_timeout=0; + server unix:///var/run/twitcher.socket fail_timeout=0; } # https server # http://nginx.org/en/docs/http/configuring_https_servers.html server { - listen ${https_port} ssl; - server_name ${hostname}; + listen 443 ssl; + server_name localhost; ssl_certificate cert.pem; ssl_certificate_key cert.pem; ssl_protocols TLSv1 TLSv1.1 TLSv1.2; ssl_ciphers HIGH:!aNULL:!MD5; ssl_session_cache shared:SSL:1m; ssl_session_timeout 1m; - ssl_client_certificate ${ssl_client_certificate}; + ssl_client_certificate esgf-ca-bundle.crt; #ssl_crl ca.crl; - ssl_verify_client ${ssl_verify_client}; + ssl_verify_client optional; ssl_verify_depth 2; # app diff --git a/readthedocs.yml b/readthedocs.yml new file mode 100644 index 00000000..65312d93 --- /dev/null +++ b/readthedocs.yml @@ -0,0 +1,12 @@ +version: 2 +build: + image: latest + +python: + version: 3 + install: + - method: setuptools + path: . + +conda: + environment: environment.yml diff --git a/requirements.txt b/requirements.txt index 015b996a..6da9e07d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,9 @@ +plaster_pastedeploy +waitress pyramid>=1.10 pyramid_rpc +pyramid_debugtoolbar +pyramid_retry webob requests requests_oauthlib diff --git a/requirements_dev.txt b/requirements_dev.txt index c06ff9ad..363e119e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,10 @@ -# test +-r requirements.txt +# tests pytest -mock WebTest +mock flake8 +# docs +sphinx>=1.7 +# release +bumpversion diff --git a/requirements_rtd.txt b/requirements_rtd.txt deleted file mode 100644 index 0516caff..00000000 --- a/requirements_rtd.txt +++ /dev/null @@ -1,4 +0,0 @@ -# sphinx -sphinx>=1.3 -sphinx-autoapi --e git+https://github.com/huard/sphinx-autodoc-pywps.git#egg=sphinx_autodoc_pywps diff --git a/setup.cfg b/setup.cfg index 9f63acb2..2250aa9d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,25 +1,40 @@ +[bumpversion] +current_version = 0.4.0 +commit = True +tag = True + +[metadata] +description-file = README.rst + +[bumpversion:file:twitcher/__version__.py] +search = __version__ = '{current_version}' +replace = __version__ = '{new_version}' + +[bumpversion:file:docs/source/conf.py] +search = release = '{current_version}' +replace = release = '{new_version}' + [tool:pytest] addopts = - --strict - --tb=native + --strict + --tb=native + tests/ python_files = test_*.py markers = - online: mark test to need internet connection - slow: mark test to be slow + online: mark test to need internet connection + slow: mark test to be slow [flake8] -ignore=F401,E402 -max-line-length=120 +ignore = F401,E402 +max-line-length = 120 exclude = - src, - tests, - .git, - __pycache__, - docs, - build, - dist, - bin, - eggs, - parts, - examples, - bootstrap-buildout.py, + .git, + __pycache__, + docs/source/conf.py, + build, + dist, + src, + +[doc8] +ignore-path = docs/build,docs/source/_templates,docs/source/_static +max-line-length = 120 diff --git a/setup.py b/setup.py index 0dc6083b..9da8318a 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,19 @@ import os from setuptools import setup, find_packages -version = __import__('twitcher').__version__ - here = os.path.abspath(os.path.dirname(__file__)) README = open(os.path.join(here, 'README.rst')).read() CHANGES = open(os.path.join(here, 'CHANGES.rst')).read() +about = {} +with open(os.path.join(here, 'twitcher', '__version__.py'), 'r') as f: + exec(f.read(), about) + reqs = [line.strip() for line in open('requirements.txt')] extra_reqs = [line.strip() for line in open('requirements_dev.txt')] setup(name='pyramid_twitcher', - version=version, + version=about['__version__'], description='Security Proxy for OGC Services like WPS.', long_description=README + '\n\n' + CHANGES, classifiers=[ @@ -21,8 +23,8 @@ "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", "Development Status :: 4 - Beta", ], - author='Carsten Ehbrecht', - author_email='ehbrecht@dkrz.de', + author=about['__author__'], + author_email=about['__email__'], url='https://github.com/bird-house/twitcher.git', license='Apache License 2.0', keywords='pyramid twitcher birdhouse wps security proxy ows ogc', diff --git a/spec-file.txt b/spec-file.txt new file mode 100644 index 00000000..daf9e11d --- /dev/null +++ b/spec-file.txt @@ -0,0 +1,36 @@ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +@EXPLICIT +https://repo.anaconda.com/pkgs/main/linux-64/ca-certificates-2019.1.23-0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-8.2.0-hdf63c60_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-8.2.0-hdf63c60_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/icu-58.2-h9c2bf20_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/libffi-3.2.1-hd88cf55_4.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/ncurses-6.1-he6710b0_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/openssl-1.1.1b-h7b6447c_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/xz-5.2.4-h14c3975_4.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/zlib-1.2.11-h7b6447c_3.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/libedit-3.1.20181209-hc058e9b_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/libxml2-2.9.9-he19cac6_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/readline-7.0-h7b6447c_5.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/tk-8.6.8-hbc83047_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/libxslt-1.1.33-h7d1a2b0_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/sqlite-3.28.0-h7b6447c_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/python-3.7.3-h0371630_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/asn1crypto-0.24.0-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/certifi-2019.3.9-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/chardet-3.0.4-py37_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/idna-2.8-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/lxml-4.3.3-py37hefd8a0e_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/pycparser-2.19-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/pysocks-1.6.8-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/six-1.12.0-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/cffi-1.12.3-py37h2e261b9_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/setuptools-41.0.1-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/cryptography-2.6.1-py37h1ba5d50_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/wheel-0.33.1-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/pip-19.1-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/pyopenssl-19.0.0-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/urllib3-1.24.2-py37_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/requests-2.21.0-py37_0.tar.bz2 diff --git a/templates/twitcher.ini b/templates/twitcher.ini deleted file mode 100644 index d9aed2fa..00000000 --- a/templates/twitcher.ini +++ /dev/null @@ -1,71 +0,0 @@ -### -# app configuration -# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html -### - -[app:main] -use = egg:pyramid_twitcher - -pyramid.reload_templates = false -pyramid.debug_authorization = false -pyramid.debug_notfound = false -pyramid.debug_routematch = false -pyramid.default_locale_name = en - -# mongodb -mongodb.host = localhost -mongodb.port = ${settings:mongodb-port} -mongodb.db_name = twitcher_db - -# twitcher -twitcher.url = ${settings:twitcher-url} -twitcher.rcpinterface = ${settings:rpcinterface} -twitcher.username = ${settings:username} -twitcher.password = ${settings:password} -twitcher.ows_security = ${settings:ows-security} -twitcher.ows_proxy = ${settings:ows-proxy} -twitcher.ows_proxy_delegate = ${settings:ows-proxy-delegate} -twitcher.workdir = ${settings:workdir} -twitcher.prefix = ${settings:workdir-prefix} -twitcher.ows_proxy_protected_path = ${settings:ows-proxy-protected-path} - -### -# wsgi server configuration -### - -[server:main] -use = egg:gunicorn#main -bind=unix://${socket} -workers=${workers} - -### -# logging configuration -# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html -### - -[loggers] -keys = root, twitcher - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = ${settings:log-level} -handlers = console - -[logger_twitcher] -level = ${settings:log-level} -handlers = console -qualname = twitcher - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = DEBUG -formatter = generic - -[formatter_generic] -format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s diff --git a/tests/functional/common.py b/tests/functional/common.py index a0c05634..6297f3df 100644 --- a/tests/functional/common.py +++ b/tests/functional/common.py @@ -4,18 +4,13 @@ from twitcher.store import tokenstore_factory from twitcher.store import servicestore_factory -import six -if six.PY2: - import xmlrpclib -else: - import xmlrpc.client as xmlrpclib - +import xmlrpc.client as xmlrpclib WPS_TEST_SERVICE = 'http://localhost:5000/wps' def setup_with_mongodb(): - settings = {'mongodb.host': '127.0.0.1', 'mongodb.port': '27027', 'mongodb.db_name': 'twitcher_test'} + settings = {'mongodb.host': '127.0.0.1', 'mongodb.port': '27017', 'mongodb.db_name': 'twitcher_test'} config = testing.setUp(settings=settings) return config @@ -35,10 +30,7 @@ def setup_mongodb_servicestore(config): def call_FUT(app, method, params): - if six.PY2: - xml = xmlrpclib.dumps(params, methodname=method) - else: - xml = xmlrpclib.dumps(params, methodname=method).encode('utf-8') + xml = xmlrpclib.dumps(params, methodname=method).encode('utf-8') print(xml) resp = app.post('/RPC2', content_type='text/xml', params=xml) assert resp.status_int == 200 diff --git a/tests/test_datatype.py b/tests/test_datatype.py index d085a482..f3705824 100644 --- a/tests/test_datatype.py +++ b/tests/test_datatype.py @@ -23,7 +23,7 @@ def test_access_token(self): assert 'expires_at' in access_token.params def test_missing_token(self): - with pytest.raises(TypeError) as e_info: + with pytest.raises(TypeError): AccessToken() def test_invalid_access_token(self): @@ -45,7 +45,7 @@ def test_service_with_url_only(self): assert service.has_purl() is False def test_missing_url(self): - with pytest.raises(TypeError) as e_info: + with pytest.raises(TypeError): Service() def test_service_with_name(self): diff --git a/tests/test_owsrequest_wms.py b/tests/test_owsrequest_wms.py index ae4136e4..69796081 100644 --- a/tests/test_owsrequest_wms.py +++ b/tests/test_owsrequest_wms.py @@ -47,5 +47,3 @@ def test_get_getmetadata_request(self): assert ows_req.request == 'getmetadata' assert ows_req.service == 'wms' assert ows_req.version == '1.3.0' - - diff --git a/tests/test_owsrequest_wps.py b/tests/test_owsrequest_wps.py index 7b19aee9..28a6f128 100644 --- a/tests/test_owsrequest_wps.py +++ b/tests/test_owsrequest_wps.py @@ -42,26 +42,26 @@ def test_get_execute_request(self): def test_get_false_request(self): params = dict(request="tellmemore", service="Wps", version="1.0.0") request = DummyRequest(params=params) - with pytest.raises(OWSInvalidParameterValue) as e_info: - ows_req = OWSRequest(request) + with pytest.raises(OWSInvalidParameterValue): + OWSRequest(request) def test_get_missing_request(self): params = dict(service="wps", version="1.0.0") request = DummyRequest(params=params) - with pytest.raises(OWSMissingParameterValue) as e_info: - ows_req = OWSRequest(request) + with pytest.raises(OWSMissingParameterValue): + OWSRequest(request) def test_get_false_service(self): params = dict(request="execute", service="ATM", version="1.0.0") request = DummyRequest(params=params) - with pytest.raises(OWSInvalidParameterValue) as e_info: - ows_req = OWSRequest(request) + with pytest.raises(OWSInvalidParameterValue): + OWSRequest(request) def test_get_missing_service(self): params = dict(request="Execute", version="1.0.0") request = DummyRequest(params=params) - with pytest.raises(OWSMissingParameterValue) as e_info: - ows_req = OWSRequest(request) + with pytest.raises(OWSMissingParameterValue): + OWSRequest(request) def test_post_getcaps_request(self): request = DummyRequest(post={}) @@ -75,15 +75,15 @@ def test_post_false_request(self): request = DummyRequest(post={}) request.body = b""" """ - with pytest.raises(OWSInvalidParameterValue) as e_info: - ows_req = OWSRequest(request) + with pytest.raises(OWSInvalidParameterValue): + OWSRequest(request) def test_post_false_service(self): request = DummyRequest(post={}) request.body = b""" """ - with pytest.raises(OWSInvalidParameterValue) as e_info: - ows_req = OWSRequest(request) + with pytest.raises(OWSInvalidParameterValue): + OWSRequest(request) def test_post_describeprocess_request(self): request = DummyRequest(post={}) diff --git a/tests/test_owssecurity.py b/tests/test_owssecurity.py index 80f2ce4c..8b2a3173 100644 --- a/tests/test_owssecurity.py +++ b/tests/test_owssecurity.py @@ -60,7 +60,7 @@ def test_check_request_invalid(self): request = DummyRequest(params=params, path='/ows/proxy/emu') request.registry = Registry() request.registry.settings = {'twitcher.ows_prox_protected_path': '/ows'} - with pytest.raises(OWSAccessForbidden) as e_info: + with pytest.raises(OWSAccessForbidden): security.check_request(request) def test_check_request_allowed_caps(self): diff --git a/tests/test_utils.py b/tests/test_utils.py index 677bafe6..c837a1e5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -18,9 +18,9 @@ def test_parse_service_name(): assert 'emu' == utils.parse_service_name("/ows/proxy/emu", protected_path) assert 'emu' == utils.parse_service_name("/ows/proxy/emu/foo/bar", protected_path) assert 'emu' == utils.parse_service_name("/ows/proxy/emu/", protected_path) - with pytest.raises(ServiceNotFound) as e_info: + with pytest.raises(ServiceNotFound): assert 'emu' == utils.parse_service_name("/ows/proxy/", protected_path) - with pytest.raises(ServiceNotFound) as e_info: + with pytest.raises(ServiceNotFound): assert 'emu' == utils.parse_service_name("/ows/nowhere/emu", protected_path) @@ -29,7 +29,7 @@ def test_baseurl(): assert utils.baseurl('http://localhost:8094/wps?service=wps&request=getcapabilities') == 'http://localhost:8094/wps' assert utils.baseurl('https://localhost:8094/wps?service=wps&request=getcapabilities') ==\ 'https://localhost:8094/wps' - with pytest.raises(ValueError) as e_info: + with pytest.raises(ValueError): utils.baseurl('ftp://localhost:8094/wps') diff --git a/twitcher/__init__.py b/twitcher/__init__.py index 78186c17..2398a648 100644 --- a/twitcher/__init__.py +++ b/twitcher/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.4.0' +from .__version__ import __author__, __email__, __version__ def main(global_config, **settings): diff --git a/twitcher/__version__.py b/twitcher/__version__.py new file mode 100644 index 00000000..6750e1cd --- /dev/null +++ b/twitcher/__version__.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- + +# This information is located in its own file so that it can be loaded +# without importing the main package when its dependencies are not installed. +# See: https://packaging.python.org/guides/single-sourcing-package-version + +__author__ = """Carsten Ehbrecht""" +__email__ = 'ehbrecht@dkrz.de' +__version__ = '0.4.0' diff --git a/twitcher/db.py b/twitcher/db.py index d2a5d258..d18da50a 100644 --- a/twitcher/db.py +++ b/twitcher/db.py @@ -4,9 +4,6 @@ import pymongo -import logging -logger = logging.getLogger(__name__) - def mongodb(registry): settings = registry.settings @@ -17,12 +14,12 @@ def mongodb(registry): return db -def includeme(config): - config.registry.db = mongodb(config.registry) - - def _add_db(request): - db = request.registry.db - # if db_url.username and db_url.password: - # db.authenticate(db_url.username, db_url.password) - return db - config.add_request_method(_add_db, 'db', reify=True) +# def includeme(config): +# config.registry.db = mongodb(config.registry) +# +# def _add_db(request): +# db = request.registry.db +# # if db_url.username and db_url.password: +# # db.authenticate(db_url.username, db_url.password) +# return db +# config.add_request_method(_add_db, 'db', reify=True) diff --git a/twitcher/owsproxy.py b/twitcher/owsproxy.py index 8314a768..bf259211 100644 --- a/twitcher/owsproxy.py +++ b/twitcher/owsproxy.py @@ -187,7 +187,7 @@ def includeme(config): # include twitcher config config.include('twitcher.config') # include mongodb - config.include('twitcher.db') + # config.include('twitcher.db') config.add_view(owsproxy, route_name='owsproxy') config.add_view(owsproxy, route_name='owsproxy_secured') config.add_view(owsproxy, route_name='owsproxy_extra') diff --git a/twitcher/rpcinterface.py b/twitcher/rpcinterface.py index cd8cc670..5faf779e 100644 --- a/twitcher/rpcinterface.py +++ b/twitcher/rpcinterface.py @@ -101,7 +101,7 @@ def includeme(config): # pyramid xml-rpc # http://docs.pylonsproject.org/projects/pyramid-rpc/en/latest/xmlrpc.html config.include('pyramid_rpc.xmlrpc') - config.include('twitcher.db') + # config.include('twitcher.db') config.add_xmlrpc_endpoint('api', '/RPC2') # register xmlrpc methods diff --git a/twitcher/twitcherctl.py b/twitcher/twitcherctl.py index e9c61f89..56ae4497 100644 --- a/twitcher/twitcherctl.py +++ b/twitcher/twitcherctl.py @@ -25,8 +25,8 @@ def create_parser(self): action="store_true") parser.add_argument('-s', '--serverurl', metavar='URL', - default='https://localhost:8000', - help='URL on which twitcher server is listening (default "https://localhost:8000").') + default='http://localhost:8000', + help='URL on which twitcher server is listening (default "http://localhost:8000").') parser.add_argument("-u", "--username", help="Username to use for authentication with server.") parser.add_argument("-p", "--password",