From 1f4b86edb3ca38421b455869c59577e18c6d8a9c Mon Sep 17 00:00:00 2001 From: Leopold Talirz Date: Mon, 22 Oct 2018 13:36:36 +0200 Subject: [PATCH] AiiDA conda environment (#2081) * add an `environment.yml` file for installing AiiDA's dependencies using conda * add a pre-commit hook that updates this file based on `setup_requirements.py` * add a Travis test that tests the conda install and prohibits adding new dependencies if they are not available on conda (either the `defaults` or the `conda-forge` channel`) * update reentry to 1.2.2, which is now available on `conda-forge` Note: Using 'defaults' channel (anaconda channel is currently breaking installs) https://github.com/conda/conda/issues/7872 --- .pre-commit-config.yaml | 15 ++++++++-- .travis-data/install_conda.sh | 14 ++++++++++ .travis-data/setup_profiles.sh | 3 +- .travis-data/test_script.sh | 11 +++++++- .travis.yml | 1 + docs/requirements_for_rtd.txt | 2 +- environment.yml | 47 +++++++++++++++++++++++++++++++ pyproject.toml | 2 +- setup_requirements.py | 2 +- utils/validate_pyproject.py | 51 ++++++++++++++++++++++++++++++---- 10 files changed, 134 insertions(+), 14 deletions(-) create mode 100755 .travis-data/install_conda.sh create mode 100644 environment.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c65ad16d3..c5911b6583 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,12 +58,23 @@ hooks: - id: pyproject name: Validating pyproject.toml - entry: python ./utils/validate_pyproject.py + entry: python ./utils/validate_pyproject.py version language: system files: >- (?x)^( setup_requirements.py| - utils/validate_pyproject| + utils/validate_pyproject.py| + )$ + pass_filenames: false + - id: conda + name: Validating environment.yml + entry: python ./utils/validate_pyproject.py conda + language: system + files: >- + (?x)^( + setup_requirements.py| + utils/validate_pyproject.py| + environment.yml| )$ pass_filenames: false diff --git a/.travis-data/install_conda.sh b/.travis-data/install_conda.sh new file mode 100755 index 0000000000..e09cef5523 --- /dev/null +++ b/.travis-data/install_conda.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# See https://conda.io/docs/user-guide/tasks/use-conda-with-travis-ci.html#the-travis-yml-file +if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then + wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; +else + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; +fi +bash miniconda.sh -b -p $HOME/miniconda +export PATH="$HOME/miniconda/bin:$PATH" +hash -r +conda config --set always_yes yes --set changeps1 no +conda update -q conda +# Useful for debugging any issues with conda +conda info -a diff --git a/.travis-data/setup_profiles.sh b/.travis-data/setup_profiles.sh index dc63842749..99c7d5a2a6 100755 --- a/.travis-data/setup_profiles.sh +++ b/.travis-data/setup_profiles.sh @@ -1,7 +1,6 @@ set -ev -if [[ "$TEST_TYPE" != "pre-commit" ]] - # no setup at all required for pre-commit to run +if [[ "$TEST_TYPE" == "tests" || "$TEST_TYPE" == "docs" ]] then # Here I create the actual DB for submission psql -c "CREATE DATABASE $TEST_AIIDA_BACKEND;" -U postgres diff --git a/.travis-data/test_script.sh b/.travis-data/test_script.sh index 2d690d7d10..c5e4124aa6 100755 --- a/.travis-data/test_script.sh +++ b/.travis-data/test_script.sh @@ -3,6 +3,8 @@ # Be verbose, and stop with error as soon there's one set -ev +DATA_DIR=${TRAVIS_BUILD_DIR}/.travis-data + case "$TEST_TYPE" in docs) # Compile the docs (HTML format); @@ -12,7 +14,6 @@ case "$TEST_TYPE" in SPHINXOPTS="-nW" make -C docs ;; tests) - DATA_DIR=${TRAVIS_BUILD_DIR}/.travis-data # make sure we have the correct pg_ctl in our path for pgtest, to prevent issue #1722 # this must match the version request in travis.yml @@ -37,4 +38,12 @@ case "$TEST_TYPE" in pre-commit) pre-commit run --all-files || ( git status --short ; git diff ; exit 1 ) ;; + conda) + # Note: Not added to install in order not to slow down other tests + source ${DATA_DIR}/install_conda.sh + + # Replace dep1 dep2 ... with your dependencies + conda env create -f environment.yml -n test-environment python=$TRAVIS_PYTHON_VERSION + source activate test-environment + ;; esac diff --git a/.travis.yml b/.travis.yml index 1426106fbd..723bebf578 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,6 +49,7 @@ env: - TEST_AIIDA_BACKEND=django TEST_TYPE="docs" - TEST_AIIDA_BACKEND=django TEST_TYPE="tests" - TEST_AIIDA_BACKEND=sqlalchemy TEST_TYPE="tests" + - TEST_TYPE="conda" before_script: diff --git a/docs/requirements_for_rtd.txt b/docs/requirements_for_rtd.txt index b477c00eaf..49f0f246b5 100644 --- a/docs/requirements_for_rtd.txt +++ b/docs/requirements_for_rtd.txt @@ -55,7 +55,7 @@ python-memcached==1.59 python-mimeparse==1.6.0 pytz==2018.4 qe-tools==1.1.0 -reentry==1.2.1 +reentry==1.2.2 seekpath==1.8.1 singledispatch >= 3.4.0.3 six==1.11.0 diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000000..e429e78e29 --- /dev/null +++ b/environment.yml @@ -0,0 +1,47 @@ +# Usage: conda env create -f environment.yml +--- +channels: +- defaults +- conda-forge +- etetoolkit +dependencies: +- reentry==1.2.2 +- python-dateutil==2.7.2 +- python-mimeparse==1.6.0 +- django==1.7.11 +- django-extensions==1.5.0 +- tzlocal==1.5.1 +- pytz==2018.4 +- PyYAML==3.12 +- six==1.11.0 +- future==0.16.0 +- pathlib2==2.3.0 +- celery==3.1.25 +- billiard==3.3.0.23 +- amqp==1.4.9 +- anyjson==0.3.3 +- plumpy==0.7.12 +- portalocker==1.1.0 +- psutil==5.4.5 +- meld3==1.0.2 +- numpy==1.14.3 +- SQLAlchemy==1.0.19 +- SQLAlchemy-Utils==0.33.0 +- alembic==0.9.9 +- ujson==1.35 +- enum34==1.1.6 +- voluptuous==0.11.1 +- aldjemy==0.8.0 +- passlib==1.7.1 +- validate_email==1.3 +- click==6.7 +- click-plugins==1.0.3 +- click-spinner==0.1.7 +- tabulate==0.8.2 +- ete3==3.1.1 +- uritools==2.1.0 +- psycopg2==2.7.4 +- paramiko==2.4.2 +- ecdsa==0.13 +- ipython<6.0 +name: aiida diff --git a/pyproject.toml b/pyproject.toml index c6ef08e8a4..8d23bce8f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] # Minimum requirements for the build system to execute. -requires = ["setuptools", "wheel", "reentry==1.2.1"] +requires = ["setuptools", "wheel", "reentry==1.2.2"] diff --git a/setup_requirements.py b/setup_requirements.py index 94a2a437fd..9f77019081 100644 --- a/setup_requirements.py +++ b/setup_requirements.py @@ -9,7 +9,7 @@ ########################################################################### install_requires = [ - 'reentry==1.2.1', + 'reentry==1.2.2', 'python-dateutil==2.7.2', 'python-mimeparse==1.6.0', 'django==1.7.11', # Upgrade to Django 1.9 does prevent AiiDA functioning diff --git a/utils/validate_pyproject.py b/utils/validate_pyproject.py index 9ce57694c7..d6b9da6a6f 100644 --- a/utils/validate_pyproject.py +++ b/utils/validate_pyproject.py @@ -13,7 +13,16 @@ import sys import toml -@click.command() +dir_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(dir_path, os.pardir)) +import setup_requirements + + +@click.group() +def cli(): + pass + +@click.command('version') def validate_pyproject(): """ Ensure that the version of reentry in setup_requirements.py and pyproject.toml are identical @@ -21,11 +30,7 @@ def validate_pyproject(): filename_pyproject = 'pyproject.toml' filename_requirements = 'setup_requirements.py' - dir_path = os.path.dirname(os.path.realpath(__file__)) toml_file = os.path.join(dir_path, os.pardir, filename_pyproject) - sys.path.append(os.path.join(dir_path, os.pardir)) - - import setup_requirements reentry_requirement = None @@ -63,5 +68,39 @@ def validate_pyproject(): sys.exit(1) +@click.command('conda') +def update_environment_yml(): + """ + Updates environment.yml file for conda. + """ + from setup_requirements import install_requires + import yaml + + # fix incompatibilities between conda and pypi + replacements = { + 'psycopg2-binary' : 'psycopg2', + 'validate-email' : 'validate_email', + } + sep = '%' # use something forbidden in conda package names + pkg_string = sep.join(install_requires) + for (pypi_pkg_name, conda_pkg_name) in iter(replacements.items()): + pkg_string = pkg_string.replace(pypi_pkg_name, conda_pkg_name) + install_requires = pkg_string.split(sep) + environment = dict( + name = 'aiida', + channels = ['defaults', 'conda-forge', 'etetoolkit'], + dependencies = install_requires, + ) + + environment_filename = 'environment.yml' + file_path = os.path.join(dir_path, os.pardir, environment_filename) + with open(file_path, 'w') as env_file: + env_file.write('# Usage: conda env create -f environment.yml\n') + yaml.dump(environment, env_file, explicit_start=True, + default_flow_style=False) + +cli.add_command(validate_pyproject) +cli.add_command(update_environment_yml) + if __name__ == '__main__': - validate_pyproject() # pylint: disable=no-value-for-parameter + cli() # pylint: disable=no-value-for-parameter