diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000000..41e9051abb --- /dev/null +++ b/.flake8 @@ -0,0 +1,10 @@ +[flake8] + +exclude = + compat.py, + hypothesis-python/src/hypothesis/vendor/*, + test_reflection.py, + test_imports.py, + hypothesis-python/tests/py2/*, + test_lambda_formatting.py +ignore = D1,D205,D209,D213,D400,D401,D999 diff --git a/.travis.yml b/.travis.yml index 643174a26e..2660d989ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,7 +65,7 @@ env: - TASK=deploy script: - - python scripts/run_travis_make_task.py + - ./build.sh matrix: fast_finish: true diff --git a/Makefile b/Makefile index 869cdbc3e5..486836067a 100644 --- a/Makefile +++ b/Makefile @@ -1,290 +1,6 @@ +# You don't need to use this Makefile and should use build.sh instead. This is +# just here so that us poor souls who remember the Make based system and keep +# typing "make target" can ease our transition to the new system. -.PHONY: clean documentation - - -ROOT_DIR:=$(shell git rev-parse --show-toplevel) - -HYPOTHESIS_PYTHON=$(ROOT_DIR)/hypothesis-python - - -DEVELOPMENT_DATABASE?=postgres://whereshouldilive@localhost/whereshouldilive_dev -SPHINXBUILD = $(DEV_PYTHON) -m sphinx -SPHINX_BUILDDIR = docs/_build -ALLSPHINXOPTS = -d $(SPHINX_BUILDDIR)/doctrees docs -W - -export BUILD_RUNTIMES?=$(HOME)/.cache/hypothesis-build-runtimes -export TOX_WORK_DIR=$(BUILD_RUNTIMES)/.tox -export COVERAGE_FILE=$(BUILD_RUNTIMES)/.coverage - -PY27=$(BUILD_RUNTIMES)/snakepit/python2.7 -PY273=$(BUILD_RUNTIMES)/snakepit/python2.7.3 -PY34=$(BUILD_RUNTIMES)/snakepit/python3.4 -PY35=$(BUILD_RUNTIMES)/snakepit/python3.5 -PY36=$(BUILD_RUNTIMES)/snakepit/python3.6 -PYPY=$(BUILD_RUNTIMES)/snakepit/pypy - -BEST_PY3=$(PY36) - -TOOLS=$(BUILD_RUNTIMES)/tools - -TOX=$(TOOLS)/tox -SPHINX_BUILD=$(TOOLS)/sphinx-build -ISORT=$(TOOLS)/isort -FLAKE8=$(TOOLS)/flake8 -PYFORMAT=$(TOOLS)/pyformat -RSTLINT=$(TOOLS)/rst-lint -PIPCOMPILE=$(TOOLS)/pip-compile - -TOOL_VIRTUALENV:=$(BUILD_RUNTIMES)/virtualenvs/tools-$(shell scripts/tool-hash.py tools) - -TOOL_PYTHON=$(TOOL_VIRTUALENV)/bin/python -TOOL_PIP=$(TOOL_VIRTUALENV)/bin/pip - -FILES_TO_FORMAT=$(BEST_PY3) scripts/files-to-format.py - - -define run_tox - cd $(HYPOTHESIS_PYTHON); $(TOX) --recreate -e $(1) -endef - - -export PATH:=$(BUILD_RUNTIMES)/snakepit:$(TOOLS):$(PATH) -export LC_ALL=en_US.UTF-8 - -$(PY27): - scripts/retry.sh scripts/install.sh 2.7 - -$(PY273): - scripts/retry.sh scripts/install.sh 2.7.3 - -$(PY34): - scripts/retry.sh scripts/install.sh 3.4 - -$(PY35): - scripts/retry.sh scripts/install.sh 3.5 - -$(PY36): - scripts/retry.sh scripts/install.sh 3.6 - - -$(PYPY): - scripts/retry.sh scripts/install.sh pypy - -$(TOOL_VIRTUALENV): $(BEST_PY3) - $(BEST_PY3) -m virtualenv $(TOOL_VIRTUALENV) - $(TOOL_PIP) install -r requirements/tools.txt - -$(TOOLS): $(TOOL_VIRTUALENV) - mkdir -p $(TOOLS) - -install-tools: $(TOOLS) - -format: $(PYFORMAT) $(ISORT) - $(FILES_TO_FORMAT) | xargs $(TOOL_PYTHON) scripts/enforce_header.py - # isort will sort packages differently depending on whether they're installed - $(FILES_TO_FORMAT) | xargs env -i PATH="$(PATH)" $(ISORT) -p hypothesis -ls -m 2 -w 75 \ - -a "from __future__ import absolute_import, print_function, division" - $(FILES_TO_FORMAT) | xargs $(PYFORMAT) -i - -lint: $(FLAKE8) - $(FLAKE8) src tests - - -check-pyup-yml: $(TOOL_VIRTUALENV) - $(TOOL_PYTHON) scripts/validate_pyup.py - -check-release-file: $(BEST_PY3) - $(BEST_PY3) scripts/check-release-file.py - -deploy: $(TOOL_VIRTUALENV) - $(TOOL_PYTHON) scripts/deploy.py - -check-format: format - find src tests -name "*.py" | xargs $(TOOL_PYTHON) scripts/check_encoding_header.py - git diff --exit-code - -install-core: $(PY27) $(PYPY) $(BEST_PY3) $(TOX) - -STACK=$(HOME)/.local/bin/stack -GHC=$(HOME)/.local/bin/ghc -SHELLCHECK=$(HOME)/.local/bin/shellcheck - -$(STACK): - mkdir -p ~/.local/bin - curl -L https://www.stackage.org/stack/linux-x86_64 | tar xz --wildcards --strip-components=1 -C $(HOME)/.local/bin '*/stack' - -$(GHC): $(STACK) - $(STACK) setup - -$(SHELLCHECK): $(GHC) - $(STACK) install ShellCheck - -check-shellcheck: $(SHELLCHECK) - shellcheck scripts/*.sh - -check-py27: $(PY27) $(TOX) - $(call run_tox,py27-full) - -check-py273: $(PY273) $(TOX) - $(call run_tox,oldpy27) - -check-py27-typing: $(PY27) $(TOX) - $(call run_tox,py27typing) - -check-py34: $(PY34) $(TOX) - $(call run_tox,py34-full) - -check-py35: $(PY35) $(TOX) - $(call run_tox,py35-full) - -check-py36: $(BEST_PY3) $(TOX) - $(call run_tox,py36-full) - -check-pypy: $(PYPY) $(TOX) - $(call run_tox,pypy-full) - -check-pypy-with-tracer: $(PYPY) $(TOX) - $(call run_tox,pypy-with-tracer) - -check-nose: $(TOX) - $(call run_tox,nose) - -check-pytest30: $(TOX) - $(call run_tox,pytest30) - -check-pytest28: $(TOX) - $(call run_tox,pytest28) - -check-quality: $(TOX) $(PY27) - $(call run_tox,quality) - $(call run_tox,quality2) - -check-ancient-pip: $(PY273) - scripts/check-ancient-pip.sh $(PY273) - - -check-pytest: check-pytest28 check-pytest30 - -check-faker070: $(TOX) - $(call run_tox,faker070) - -check-faker-latest: $(TOX) - $(call run_tox,faker-latest) - -check-django111: $(TOX) - $(call run_tox,django111) - -check-django20: $(BEST_PY3) $(TOX) - $(call run_tox,django20) - -check-django: check-django111 check-django20 - -check-pandas19: $(TOX) - $(call run_tox,pandas19) - -check-pandas20: $(TOX) - $(call run_tox,pandas20) - -check-pandas21: $(TOX) - $(call run_tox,pandas21) - -check-pandas22: $(TOX) - $(call run_tox,pandas22) - -check-examples2: $(TOX) $(PY27) - $(call run_tox,examples2) - -check-examples3: $(TOX) - $(call run_tox,examples3) - -check-coverage: $(TOX) - $(call run_tox,coverage) - -check-pure-tracer: $(TOX) - $(call run_tox,pure-tracer) - -check-unicode: $(TOX) $(PY27) - $(call run_tox,unicode) - -check-noformat: check-coverage check-py26 check-py27 check-py34 check-py35 check-pypy check-django check-pytest - -check: check-format check-noformat - -check-fast: lint $(PYPY) $(PY36) $(TOX) - $(call run_tox,pypy-brief) - $(call run_tox,py36-prettyquick) - -check-rst: $(RSTLINT) $(FLAKE8) - $(RSTLINT) CONTRIBUTING.rst README.rst - $(RSTLINT) guides/*.rst - $(FLAKE8) --select=W191,W291,W292,W293,W391 *.rst hypothesis-python/*.rst hypothesis-python/docs/*.rst - -compile-requirements: $(PIPCOMPILE) - $(PIPCOMPILE) requirements/test.in --output-file requirements/test.txt - $(PIPCOMPILE) requirements/tools.in --output-file requirements/tools.txt - $(PIPCOMPILE) requirements/typing.in --output-file requirements/typing.txt - $(PIPCOMPILE) requirements/coverage.in --output-file requirements/coverage.txt - -upgrade-requirements: - $(PIPCOMPILE) --upgrade requirements/test.in --output-file requirements/test.txt - $(PIPCOMPILE) --upgrade requirements/tools.in --output-file requirements/tools.txt - $(PIPCOMPILE) --upgrade requirements/typing.in --output-file requirements/typing.txt - $(PIPCOMPILE) --upgrade requirements/coverage.in --output-file requirements/coverage.txt - -check-requirements: compile-requirements - git diff --exit-code - -secrets.tar.enc: deploy_key .pypirc - rm -f secrets.tar secrets.tar.enc - tar -cf secrets.tar deploy_key .pypirc - travis encrypt-file secrets.tar - rm secrets.tar - -$(TOX): $(BEST_PY3) $(HYPOTHESIS_PYTHON)/tox.ini $(TOOLS) - rm -f $(TOX) - ln -sf $(TOOL_VIRTUALENV)/bin/tox $(TOX) - touch $(TOOL_VIRTUALENV)/bin/tox $(TOX) - -$(SPHINX_BUILD): $(TOOLS) - ln -sf $(TOOL_VIRTUALENV)/bin/sphinx-build $(SPHINX_BUILD) - -$(PYFORMAT): $(TOOLS) - ln -sf $(TOOL_VIRTUALENV)/bin/pyformat $(PYFORMAT) - -$(ISORT): $(TOOLS) - ln -sf $(TOOL_VIRTUALENV)/bin/isort $(ISORT) - -$(RSTLINT): $(TOOLS) - ln -sf $(TOOL_VIRTUALENV)/bin/rst-lint $(RSTLINT) - -$(FLAKE8): $(TOOLS) - ln -sf $(TOOL_VIRTUALENV)/bin/flake8 $(FLAKE8) - -$(PIPCOMPILE): $(TOOLS) - ln -sf $(TOOL_VIRTUALENV)/bin/pip-compile $(PIPCOMPILE) - - -clean: - rm -rf .tox - rm -rf .hypothesis - rm -rf docs/_build - rm -rf $(TOOLS) - rm -rf $(BUILD_RUNTIMES)/snakepit - rm -rf $(BUILD_RUNTIMES)/virtualenvs - find src tests -name "*.pyc" -delete - find src tests -name "__pycache__" -delete - -.PHONY: RELEASE.rst -RELEASE.rst: - -documentation: $(SPHINX_BUILD) - scripts/build-documentation.sh $(SPHINX_BUILD) $(PY36) - -doctest: $(SPHINX_BUILD) - cd $(HYPOTHESIS_PYTHON); PYTHONPATH=src $(SPHINX_BUILD) -W -b doctest -d docs/_build/doctrees docs docs/_build/html - -fix_doctests: $(TOOL_VIRTUALENV) - PYTHONPATH=src $(TOOL_PYTHON) scripts/fix_doctests.py - -check-secrets: $(TOOL_VIRTUALENV) - $(TOOL_PYTHON) scripts/check_secrets.py +%: + ./build.sh $@ diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000..61bb4f2861 --- /dev/null +++ b/build.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# This script is here to bootstrap the Hypothesis build process into a working +# version of Python, then hand over to the actual Hypothesis build runner (which +# is written in Python instead of bash). + +set -o xtrace +set -o errexit +set -o nounset + +ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel)" + +export HYPOTHESIS_ROOT="$ROOT" + +SCRIPTS="$ROOT/tooling/scripts" + +# shellcheck source=tooling/scripts/common.sh +source "$SCRIPTS/common.sh" + +"$SCRIPTS/ensure-python.sh" 3.6.5 + +PYTHON=$(pythonloc 3.6.5)/bin/python + +TOOL_REQUIREMENTS="$ROOT/requirements/tools.txt" + +TOOL_HASH=$("$PYTHON" "$SCRIPTS/tool-hash.py" < "$TOOL_REQUIREMENTS") + +TOOL_VIRTUALENV="$VIRTUALENVS/build-$TOOL_HASH" +TOOL_PYTHON="$TOOL_VIRTUALENV/bin/python" + +if [ ! -e "$TOOL_PYTHON" ] ; then + rm -rf "$TOOL_VIRTUALENV" + "$PYTHON" -m pip install --upgrade virtualenv + "$PYTHON" -m virtualenv "$TOOL_VIRTUALENV" + "$TOOL_PYTHON" -m pip install --no-warn-script-location -r requirements/tools.txt + "$TOOL_PYTHON" -m pip install -e tooling +fi + +"$TOOL_PYTHON" -m hypothesistooling "$@" diff --git a/circle.yml b/circle.yml index 37b23225c1..fdb34f4702 100644 --- a/circle.yml +++ b/circle.yml @@ -1,19 +1,16 @@ test: override: - - scripts/run_circle.py + - ./build.sh check-py27 + - ./build.sh check-py36 machine: pre: - brew update - - brew install python - - ln -f -s $(which python3) /usr/local/bin/python - - ln -f -s $(which pip3) /usr/local/bin/pip - - python --version - - pip --version + - brew install readline xz ncurses dependencies: override: - - make install-core + - ./build.sh install-core cache_directories: - ~/.cache/hypothesis-build-runtimes diff --git a/guides/internals.rst b/guides/internals.rst index 3187fa5f13..9fd6822423 100644 --- a/guides/internals.rst +++ b/guides/internals.rst @@ -3,7 +3,7 @@ How to Work on Hypothesis Internals =================================== Note: Currently this guide is very specific to the *Python* version of Hypothesis. -Over time the core will be factored out into a small separate set of libraries - +Over time the core will be factored out into a small separate set of libraries - the current migration plan is to move all of the Python code into Rust and have the Python and Ruby versions both depend on this. Eventually we will likely need to have more than one core library - e.g. a Java one as well. diff --git a/guides/review.rst b/guides/review.rst index 7c04ec2693..90e72808ce 100644 --- a/guides/review.rst +++ b/guides/review.rst @@ -107,7 +107,7 @@ Clarity of Description The ``RELEASE.rst`` should contain a description of the change that makes clear: -1. The motivation for the change +1. The motivation for the change 2. The likely consequences of the change This doesn't have to be an essay. If you're following the orthogonality diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..06b9d1e738 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,4 @@ +RELEASE_TYPE: patch + +This release involves some very minor internal clean up and should have no +user visible effect at all. diff --git a/scripts/basic-test.sh b/hypothesis-python/scripts/basic-test.sh similarity index 95% rename from scripts/basic-test.sh rename to hypothesis-python/scripts/basic-test.sh index 41dd925d8c..26375f831e 100755 --- a/scripts/basic-test.sh +++ b/hypothesis-python/scripts/basic-test.sh @@ -1,7 +1,8 @@ #!/bin/bash set -e -o xtrace -cd "$(dirname "$0")"/../hypothesis-python +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$HERE/.." # We run a reduced set of tests on OSX mostly so the CI runs in vaguely # reasonable time. diff --git a/scripts/unicodechecker.py b/hypothesis-python/scripts/unicodechecker.py similarity index 94% rename from scripts/unicodechecker.py rename to hypothesis-python/scripts/unicodechecker.py index bc4d0699a7..17ae2623a3 100644 --- a/scripts/unicodechecker.py +++ b/hypothesis-python/scripts/unicodechecker.py @@ -26,7 +26,6 @@ import unicodenazi from hypothesis import settings, unlimited from hypothesis.errors import HypothesisDeprecationWarning -from hypothesistooling import PYTHON_TESTS from hypothesis.configuration import set_hypothesis_home_dir warnings.filterwarnings('error', category=UnicodeWarning) @@ -48,12 +47,17 @@ 'test_testdecorators', ] -sys.path.append(os.path.join(PYTHON_TESTS, 'cover')) +sys.path.append(os.path.join('tests', 'cover')) -if __name__ == '__main__': + +def main(): for t in TESTS: module = __import__(t) for k, v in sorted(module.__dict__.items(), key=lambda x: x[0]): if k.startswith('test_') and inspect.isfunction(v): print(k) v() + + +if __name__ == '__main__': + main() diff --git a/scripts/validate_branch_check.py b/hypothesis-python/scripts/validate_branch_check.py similarity index 100% rename from scripts/validate_branch_check.py rename to hypothesis-python/scripts/validate_branch_check.py diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py index c468e17aab..19015da52c 100644 --- a/hypothesis-python/src/hypothesis/_settings.py +++ b/hypothesis-python/src/hypothesis/_settings.py @@ -24,7 +24,6 @@ from __future__ import division, print_function, absolute_import import os -import inspect import warnings import threading from enum import Enum, IntEnum, unique diff --git a/hypothesis-python/tests/cover/test_given_error_conditions.py b/hypothesis-python/tests/cover/test_given_error_conditions.py index c7efcf7130..a4fccc4372 100644 --- a/hypothesis-python/tests/cover/test_given_error_conditions.py +++ b/hypothesis-python/tests/cover/test_given_error_conditions.py @@ -17,12 +17,10 @@ from __future__ import division, print_function, absolute_import -import time - import pytest from hypothesis import HealthCheck, given, infer, assume, reject, settings -from hypothesis.errors import Timeout, Unsatisfiable, InvalidArgument +from hypothesis.errors import Unsatisfiable, InvalidArgument from tests.common.utils import fails_with, validate_deprecation from hypothesis.strategies import booleans, integers diff --git a/hypothesis-python/tests/nocover/test_strategy_state.py b/hypothesis-python/tests/nocover/test_strategy_state.py index ddb620e333..2caea00838 100644 --- a/hypothesis-python/tests/nocover/test_strategy_state.py +++ b/hypothesis-python/tests/nocover/test_strategy_state.py @@ -21,8 +21,7 @@ import hashlib from random import Random -from hypothesis import Verbosity, seed, given, assume, settings, unlimited -from hypothesis.errors import FailedHealthCheck +from hypothesis import Verbosity, assume, settings, unlimited from hypothesis.database import ExampleDatabase from hypothesis.stateful import Bundle, RuleBasedStateMachine, rule from hypothesis.strategies import data, just, none, text, lists, binary, \ diff --git a/hypothesis-python/tox.ini b/hypothesis-python/tox.ini index 4682400d39..d8ee9f8140 100644 --- a/hypothesis-python/tox.ini +++ b/hypothesis-python/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{27,34,35,36,py}-{brief,prettyquick,full,custom} +envlist = py{27,34,35,36,py27}-{brief,prettyquick,full,custom} toxworkdir={env:TOX_WORK_DIR:.tox} [testenv] @@ -15,7 +15,7 @@ passenv= setenv= brief: HYPOTHESIS_PROFILE=speedy commands = - full: bash ../scripts/basic-test.sh + full: bash scripts/basic-test.sh brief: python -m pytest tests/cover/test_testdecorators.py {posargs} prettyquick: python -m pytest tests/cover/ custom: python -m pytest {posargs} @@ -34,13 +34,6 @@ deps= commands= python -m pytest tests/quality/ -[testenv:oldpy27] -basepython=python2.7.3 -deps= - -r../requirements/test.txt -commands= - python -m pytest tests/cover/ -n2 - [testenv:py27typing] basepython=python2.7 deps= @@ -57,7 +50,7 @@ setenv= UNICODENAZI=true PYTHONPATH=. commands= - python ../scripts/unicodechecker.py + python scripts/unicodechecker.py [testenv:faker070] deps = @@ -146,7 +139,7 @@ commands = python -m coverage debug sys python -m coverage run --rcfile=.coveragerc -m pytest -n0 --strict tests/cover tests/datetime tests/py3 tests/numpy tests/pandas --maxfail=1 --ff {posargs} python -m coverage report -m --fail-under=100 --show-missing - python ../scripts/validate_branch_check.py + python scripts/validate_branch_check.py [testenv:pure-tracer] @@ -181,16 +174,3 @@ deps= -r../requirements/test.txt commands= python -m pytest examples - -[pytest] -addopts=--strict --tb=native -p pytester --runpytest=subprocess --durations=20 - -[flake8] -exclude = - compat.py, - src/hypothesis/vendor/, - test_reflection.py, - test_imports.py, - tests/py2, - test_lambda_formatting.py -ignore = D1,D205,D209,D213,D400,D401,D999 diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..6e9345a165 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] + +addopts=--strict --tb=native -p pytester --runpytest=subprocess --durations=20 diff --git a/scripts/build-documentation.sh b/scripts/build-documentation.sh deleted file mode 100755 index 8ca917120a..0000000000 --- a/scripts/build-documentation.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -u -set -x - -SPHINX_BUILD=$1 -PYTHON=$2 - -HERE="$(dirname "$0")" - -cd "$HERE"/../hypothesis-python - -if [ -e RELEASE.rst ] ; then - trap "git checkout docs/changes.rst src/hypothesis/version.py" EXIT - $PYTHON ../scripts/update-changelog-for-docs.py -fi - -export PYTHONPATH=src - -$SPHINX_BUILD -W -b html -d docs/_build/doctrees docs docs/_build/html diff --git a/scripts/check-ancient-pip.sh b/scripts/check-ancient-pip.sh deleted file mode 100755 index 31f79db6b2..0000000000 --- a/scripts/check-ancient-pip.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -PYTHON=$1 - -cd "$(dirname "$0")"/../hypothesis-python - -BROKEN_VIRTUALENV=$($PYTHON -c'import tempfile; print(tempfile.mkdtemp())') - -trap 'rm -rf $BROKEN_VIRTUALENV' EXIT - -rm -rf tmp-dist-dir - -$PYTHON setup.py sdist --dist-dir=tmp-dist-dir - -$PYTHON -m pip install virtualenv -$PYTHON -m virtualenv "$BROKEN_VIRTUALENV" -"$BROKEN_VIRTUALENV"/bin/pip install -r../requirements/test.txt - -# These are versions from debian stable as of 2017-04-21 -# See https://packages.debian.org/stable/python/ -"$BROKEN_VIRTUALENV"/bin/python -m pip install --upgrade pip==1.5.6 -"$BROKEN_VIRTUALENV"/bin/pip install --upgrade setuptools==5.5.1 -"$BROKEN_VIRTUALENV"/bin/pip install tmp-dist-dir/* -"$BROKEN_VIRTUALENV"/bin/python -m pytest tests/cover/test_testdecorators.py diff --git a/scripts/check-release-file.py b/scripts/check-release-file.py deleted file mode 100644 index 62ccd10500..0000000000 --- a/scripts/check-release-file.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python - -# coding=utf-8 -# -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis-python -# -# Most of this work is copyright (C) 2013-2018 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - -from __future__ import division, print_function, absolute_import - -import os -import sys - -import hypothesistooling as tools - -sys.path.append(os.path.dirname(__file__)) # noqa - -DO_RELEASE_CHECK = os.environ.get('TRAVIS_BRANCH', 'master') == 'master' - -if __name__ == '__main__': - if DO_RELEASE_CHECK and tools.has_source_changes(): - if not tools.has_release(): - print( - 'There are source changes but no RELEASE.rst. Please create ' - 'one to describe your changes.' - ) - sys.exit(1) - tools.parse_release_file() - else: - print('Skipping release check, because this is not the master branch ' - 'or the pull request is not targeting the master branch.') diff --git a/scripts/check_encoding_header.py b/scripts/check_encoding_header.py deleted file mode 100644 index f7abc8ab83..0000000000 --- a/scripts/check_encoding_header.py +++ /dev/null @@ -1,36 +0,0 @@ -# coding=utf-8 -# -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis-python -# -# Most of this work is copyright (C) 2013-2018 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - -from __future__ import division, print_function, absolute_import - -VALID_STARTS = ( - '# coding=utf-8', - '#!/usr/bin/env python', -) - -if __name__ == '__main__': - import sys - n = max(map(len, VALID_STARTS)) - bad = False - for f in sys.argv[1:]: - with open(f, 'r', encoding='utf-8') as i: - start = i.read(n) - if not any(start.startswith(s) for s in VALID_STARTS): - print( - '%s has incorrect start %r' % (f, start), file=sys.stderr) - bad = True - sys.exit(int(bad)) diff --git a/scripts/deploy.py b/scripts/deploy.py deleted file mode 100644 index 029ab69a83..0000000000 --- a/scripts/deploy.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python - -# coding=utf-8 -# -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis-python -# -# Most of this work is copyright (C) 2013-2018 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - -from __future__ import division, print_function, absolute_import - -import os -import sys -import random -import shutil -import subprocess -from time import time, sleep - -import hypothesistooling as tools - -sys.path.append(os.path.dirname(__file__)) # noqa - - -DIST = os.path.join(tools.HYPOTHESIS_PYTHON, 'dist') - - -PENDING_STATUS = ('started', 'created') - - -if __name__ == '__main__': - os.chdir(tools.HYPOTHESIS_PYTHON) - - last_release = tools.latest_version() - - print('Current version: %s. Latest released version: %s' % ( - tools.__version__, last_release - )) - - HEAD = tools.hash_for_name('HEAD') - MASTER = tools.hash_for_name('origin/master') - print('Current head:', HEAD) - print('Current master:', MASTER) - - on_master = tools.is_ancestor(HEAD, MASTER) - has_release = tools.has_release() - - if has_release: - print('Updating changelog and version') - tools.update_for_pending_release() - - print('Building an sdist...') - - if os.path.exists(DIST): - shutil.rmtree(DIST) - - subprocess.check_output([ - sys.executable, 'setup.py', 'sdist', '--dist-dir', DIST, - ]) - - if not on_master: - print('Not deploying due to not being on master') - sys.exit(0) - - if not has_release: - print('Not deploying due to no release') - sys.exit(0) - - start_time = time() - - prev_pending = None - - # We time out after an hour, which is a stupidly long time and it should - # never actually take that long: A full Travis run only takes about 20-30 - # minutes! This is really just here as a guard in case something goes - # wrong and we're not paying attention so as to not be too mean to Travis.. - while time() <= start_time + 60 * 60: - jobs = tools.build_jobs() - - failed_jobs = [ - (k, v) - for k, vs in jobs.items() - if k not in PENDING_STATUS + ('passed',) - for v in vs - ] - - if failed_jobs: - print('Failing this due to failure of jobs %s' % ( - ', '.join('%s(%s)' % (s, j) for j, s in failed_jobs), - )) - sys.exit(1) - else: - pending = [j for s in PENDING_STATUS for j in jobs.get(s, ())] - try: - # This allows us to test the deploy job for a build locally. - pending.remove('deploy') - except ValueError: - pass - if pending: - still_pending = set(pending) - if prev_pending is None: - print('Waiting for the following jobs to complete:') - for p in sorted(still_pending): - print(' * %s' % (p,)) - print() - else: - completed = prev_pending - still_pending - if completed: - print('%s completed since last check.' % ( - ', '.join(sorted(completed)),)) - prev_pending = still_pending - naptime = 10.0 * (2 + random.random()) - print('Waiting %.2fs for %d more job%s to complete' % ( - naptime, len(pending), 's' if len(pending) > 1 else '',)) - sleep(naptime) - else: - break - else: - print("We've been waiting for an hour. That seems bad. Failing now.") - sys.exit(1) - - print('Looks good to release!') - - if os.environ.get('TRAVIS_SECURE_ENV_VARS', None) != 'true': - print("But we don't have the keys to do it") - sys.exit(0) - - print('Decrypting secrets') - tools.decrypt_secrets() - - print('Release seems good. Pushing to github now.') - - tools.create_tag_and_push() - - print('Now uploading to pypi.') - - subprocess.check_call([ - sys.executable, '-m', 'twine', 'upload', - '--config-file', tools.PYPIRC, - os.path.join(DIST, '*'), - ]) - - sys.exit(0) diff --git a/scripts/enforce_header.py b/scripts/enforce_header.py deleted file mode 100644 index 07a95cff1d..0000000000 --- a/scripts/enforce_header.py +++ /dev/null @@ -1,67 +0,0 @@ -# coding=utf-8 -# -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis-python -# -# Most of this work is copyright (C) 2013-2018 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - -from __future__ import division, print_function, absolute_import - -import os -import sys -from datetime import datetime - -HEADER_FILE = 'scripts/header.py' - -CURRENT_YEAR = datetime.utcnow().year - -HEADER_SOURCE = open(HEADER_FILE).read().strip().format(year=CURRENT_YEAR) - - -def main(): - rootdir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) - os.chdir(rootdir) - files = sys.argv[1:] - - for f in files: - print(f) - lines = [] - with open(f, encoding='utf-8') as o: - shebang = None - first = True - header_done = False - for l in o.readlines(): - if first: - first = False - if l[:2] == '#!': - shebang = l - continue - if 'END HEADER' in l and not header_done: - lines = [] - header_done = True - else: - lines.append(l) - source = ''.join(lines).strip() - with open(f, 'w', encoding='utf-8') as o: - if shebang is not None: - o.write(shebang) - o.write('\n') - o.write(HEADER_SOURCE) - if source: - o.write('\n\n') - o.write(source) - o.write('\n') - - -if __name__ == '__main__': - main() diff --git a/scripts/files-to-format.py b/scripts/files-to-format.py deleted file mode 100644 index 15f17fa1aa..0000000000 --- a/scripts/files-to-format.py +++ /dev/null @@ -1,49 +0,0 @@ -# coding=utf-8 -# -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis-python -# -# Most of this work is copyright (C) 2013-2018 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - -from __future__ import division, print_function, absolute_import - -import os - -import hypothesistooling as tools - - -def should_format_file(path): - if os.path.basename(path) in ('header.py', 'test_lambda_formatting.py'): - return False - if 'vendor' in path.split(os.path.sep): - return False - return path.endswith('.py') - - -if __name__ == '__main__': - changed = tools.modified_files() - - format_all = os.environ.get('FORMAT_ALL', '').lower() == 'true' - if 'scripts/header.py' in changed: - # We've changed the header, so everything needs its header updated. - format_all = True - if 'requirements/tools.txt' in changed: - # We've changed the tools, which includes a lot of our formatting - # logic, so we need to rerun formatters. - format_all = True - - files = tools.all_files() if format_all else changed - - for f in sorted(files): - if should_format_file(f): - print(f) diff --git a/scripts/header.py b/scripts/header.py deleted file mode 100644 index bdfba41f1a..0000000000 --- a/scripts/header.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding=utf-8 -# -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis-python -# -# Most of this work is copyright (C) 2013-{year} David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - - diff --git a/scripts/pyenv-installer b/scripts/pyenv-installer deleted file mode 100755 index dbbc5e7df6..0000000000 --- a/scripts/pyenv-installer +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -set -e -[ -n "$PYENV_DEBUG" ] && set -x - -if [ -z "$PYENV_ROOT" ]; then - PYENV_ROOT="${HOME}/.pyenv" -fi - -shell="$1" -if [ -z "$shell" ]; then - shell="$(ps c -p "$PPID" -o 'ucomm=' 2>/dev/null || true)" - shell="${shell##-}" - shell="${shell%% *}" - shell="$(basename "${shell:-$SHELL}")" -fi - -colorize() { - if [ -t 1 ]; then printf "\e[%sm%s\e[m" "$1" "$2" - else echo -n "$2" - fi -} - -checkout() { - [ -d "$2" ] || git clone "$1" "$2" -} - -if ! command -v git 1>/dev/null 2>&1; then - echo "pyenv: Git is not installed, can't continue." >&2 - exit 1 -fi - -if [ -n "${USE_HTTPS}" ]; then - GITHUB="https://github.com" -else - GITHUB="git://github.com" -fi - -checkout "${GITHUB}/yyuu/pyenv.git" "${PYENV_ROOT}" -checkout "${GITHUB}/yyuu/pyenv-doctor.git" "${PYENV_ROOT}/plugins/pyenv-doctor" -checkout "${GITHUB}/yyuu/pyenv-installer.git" "${PYENV_ROOT}/plugins/pyenv-installer" -checkout "${GITHUB}/yyuu/pyenv-pip-rehash.git" "${PYENV_ROOT}/plugins/pyenv-pip-rehash" -checkout "${GITHUB}/yyuu/pyenv-update.git" "${PYENV_ROOT}/plugins/pyenv-update" -checkout "${GITHUB}/yyuu/pyenv-virtualenv.git" "${PYENV_ROOT}/plugins/pyenv-virtualenv" -checkout "${GITHUB}/yyuu/pyenv-which-ext.git" "${PYENV_ROOT}/plugins/pyenv-which-ext" - -if ! command -v pyenv 1>/dev/null; then - { echo - colorize 1 "WARNING" - echo ": seems you still have not added 'pyenv' to the load path." - echo - } >&2 - - case "$shell" in - bash ) - profile="~/.bash_profile" - ;; - zsh ) - profile="~/.zshrc" - ;; - ksh ) - profile="~/.profile" - ;; - fish ) - profile="~/.config/fish/config.fish" - ;; - * ) - profile="your profile" - ;; - esac - - { echo "# Load pyenv automatically by adding" - echo "# the following to ${profile}:" - echo - case "$shell" in - fish ) - echo "set -x PATH \"\$HOME/.pyenv/bin\" \$PATH" - echo 'status --is-interactive; and . (pyenv init -|psub)' - echo 'status --is-interactive; and . (pyenv virtualenv-init -|psub)' - ;; - * ) - echo "export PATH=\"\$HOME/.pyenv/bin:\$PATH\"" - echo "eval \"\$(pyenv init -)\"" - echo "eval \"\$(pyenv virtualenv-init -)\"" - ;; - esac - } >&2 -fi diff --git a/scripts/retry.sh b/scripts/retry.sh deleted file mode 100755 index b9329676c0..0000000000 --- a/scripts/retry.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -for _ in $(seq 5); do - if "$@" ; then - exit 0 - fi - echo "Command failed. Retrying..." - sleep $(( ( RANDOM % 10 ) + 1 )).$(( RANDOM % 100 ))s -done - -echo "Command failed five times. Giving up now" - -exit 1 diff --git a/scripts/run_circle.py b/scripts/run_circle.py deleted file mode 100755 index f0756a48bc..0000000000 --- a/scripts/run_circle.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python - -# coding=utf-8 -# -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis-python -# -# Most of this work is copyright (C) 2013-2018 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - -from __future__ import division, print_function, absolute_import - -import os -import sys -import subprocess - -from hypothesistooling import should_run_ci_task - -if __name__ == '__main__': - - if ( - os.environ['CIRCLE_BRANCH'] != 'master' and - os.environ['CI_PULL_REQUESTS'] == '' - ): - print('We only run CI builds on the master branch or in pull requests') - sys.exit(0) - - is_pull_request = (os.environ['CI_PULL_REQUESTS'] != '') - - for task in ['check-pypy', 'check-py36', 'check-py27']: - if should_run_ci_task(task=task, is_pull_request=is_pull_request): - subprocess.check_call(['make', task]) diff --git a/scripts/run_travis_make_task.py b/scripts/run_travis_make_task.py deleted file mode 100755 index 4ecba58740..0000000000 --- a/scripts/run_travis_make_task.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -# coding=utf-8 -# -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis-python -# -# Most of this work is copyright (C) 2013-2018 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - -from __future__ import division, print_function, absolute_import - -import os -import sys -import subprocess - -from hypothesistooling import should_run_ci_task - -if __name__ == '__main__': - is_pull_request = (os.environ.get('TRAVIS_EVENT_TYPE') == 'pull_request') - task = os.environ['TASK'] - - if should_run_ci_task(task=task, is_pull_request=is_pull_request): - sys.exit(subprocess.call(['make', task])) diff --git a/scripts/tool-hash.py b/scripts/tool-hash.py deleted file mode 100755 index c5fc0ff36c..0000000000 --- a/scripts/tool-hash.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -# coding=utf-8 -# -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis-python -# -# Most of this work is copyright (C) 2013-2018 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - -from __future__ import division, print_function, absolute_import - -import os -import sys -import hashlib - -SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__)) -ROOT_DIR = os.path.dirname(SCRIPTS_DIR) - -if __name__ == '__main__': - name = sys.argv[1] - - requirements = os.path.join( - ROOT_DIR, 'requirements', '%s.txt' % (name,) - ) - - assert os.path.exists(requirements) - - with open(requirements, 'rb') as f: - tools = f.read() - print(hashlib.sha1(tools).hexdigest()[:10]) diff --git a/scripts/update-changelog-for-docs.py b/scripts/update-changelog-for-docs.py deleted file mode 100644 index 761b316728..0000000000 --- a/scripts/update-changelog-for-docs.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python - -# coding=utf-8 -# -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis-python -# -# Most of this work is copyright (C) 2013-2018 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - -from __future__ import division, print_function, absolute_import - -import os -import sys - -import hypothesistooling as tools - -sys.path.append(os.path.dirname(__file__)) # noqa - - -if __name__ == '__main__': - if not tools.has_release(): - sys.exit(0) - if tools.has_uncommitted_changes(tools.CHANGELOG_FILE): - print( - 'Cannot build documentation with uncommitted changes to ' - 'changelog and a pending release. Please commit your changes or ' - 'delete your release file.') - sys.exit(1) - tools.update_changelog_and_version() diff --git a/scripts/validate_pyup.py b/scripts/validate_pyup.py deleted file mode 100644 index 80c366dc88..0000000000 --- a/scripts/validate_pyup.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -# coding=utf-8 -# -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis-python -# -# Most of this work is copyright (C) 2013-2018 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at http://mozilla.org/MPL/2.0/. -# -# END HEADER - -from __future__ import division, print_function, absolute_import - -import os -import sys - -import yaml -from pyup.config import Config - -from hypothesistooling import ROOT - -PYUP_FILE = os.path.join(ROOT, '.pyup.yml') - -if __name__ == '__main__': - with open(PYUP_FILE, 'r') as i: - data = yaml.safe_load(i.read()) - config = Config() - config.update_config(data) - - if not config.is_valid_schedule(): - print('Schedule %r is invalid' % (config.schedule,)) - sys.exit(1) diff --git a/tooling/README.rst b/tooling/README.rst new file mode 100644 index 0000000000..7d42af830f --- /dev/null +++ b/tooling/README.rst @@ -0,0 +1,6 @@ +======================== +Hypothesis Build Tooling +======================== + +This is a piece of software for managing Hypothesis's build tasks, releases, +etc. It's very Hypothesis specific, though it may become less so in the future. diff --git a/tooling/scripts/common.sh b/tooling/scripts/common.sh new file mode 100644 index 0000000000..e992c42ac5 --- /dev/null +++ b/tooling/scripts/common.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# This file is not really a script but is intended for sourcing from other +# scripts so that they can share a common set of paths conveniently. + +set -o errexit +set -o nounset + +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT="$(git -C "$HERE" rev-parse --show-toplevel)" + +export ROOT +export BUILD_RUNTIMES=${BUILD_RUNTIMES:-$HOME/.cache/hypothesis-build-runtimes} +export BASE="$BUILD_RUNTIMES" +export PYENV="$BASE/pyenv" +export SNAKEPIT="$BASE/python-versions/" +export VIRTUALENVS="$BASE/virtualenvs/" + +pythonloc() { + VERSION="$1" + echo "$SNAKEPIT/$VERSION" +} diff --git a/tooling/scripts/ensure-python.sh b/tooling/scripts/ensure-python.sh new file mode 100755 index 0000000000..8f3a9e4592 --- /dev/null +++ b/tooling/scripts/ensure-python.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -x + +HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# shellcheck source=tooling/scripts/common.sh +source "$HERE/common.sh" + +# This is to guard against multiple builds in parallel. The various installers will tend +# to stomp all over each other if you do this and they haven't previously successfully +# succeeded. We use a lock file to block progress so only one install runs at a time. +# This script should be pretty fast once files are cached, so the loss of concurrency +# is not a major problem. +# This should be using the lockfile command, but that's not available on the +# containerized travis and we can't install it without sudo. +# It is unclear if this is actually useful. I was seeing behaviour that suggested +# concurrent runs of the installer, but I can't seem to find any evidence of this lock +# ever not being acquired. + +VERSION="$1" +TARGET=$(pythonloc "$VERSION") + +if [ ! -e "$TARGET/bin/python" ] ; then + mkdir -p "$BASE" + + LOCKFILE="$BASE/.install-lockfile" + while true; do + if mkdir "$LOCKFILE" 2>/dev/null; then + echo "Successfully acquired installer." + break + else + echo "Failed to acquire lock. Is another installer running? Waiting a bit." + fi + + sleep $(( ( RANDOM % 10 ) + 1 )).$(( RANDOM % 100 ))s + + if (( $(date '+%s') > 300 + $(stat --format=%X "$LOCKFILE") )); then + echo "We've waited long enough" + rm -rf "$LOCKFILE" + fi + done + trap 'rm -rf $LOCKFILE' EXIT + + + if [ ! -d "$PYENV/.git" ]; then + rm -rf "$PYENV" + git clone https://github.com/yyuu/pyenv.git "$PYENV" + else + back=$PWD + cd "$PYENV" + git fetch || echo "Update failed to complete. Ignoring" + git reset --hard origin/master + cd "$back" + fi + + for _ in $(seq 5); do + if "$BASE/pyenv/plugins/python-build/bin/python-build" "$VERSION" "$TARGET" ; then + exit 0 + fi + echo "Command failed. Retrying..." + sleep $(( ( RANDOM % 10 ) + 1 )).$(( RANDOM % 100 ))s + done +fi diff --git a/scripts/install.sh b/tooling/scripts/install-python.sh similarity index 100% rename from scripts/install.sh rename to tooling/scripts/install-python.sh diff --git a/scripts/check_secrets.py b/tooling/scripts/tool-hash.py old mode 100644 new mode 100755 similarity index 78% rename from scripts/check_secrets.py rename to tooling/scripts/tool-hash.py index 3bb9df1c46..f86f886dfb --- a/scripts/check_secrets.py +++ b/tooling/scripts/tool-hash.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # coding=utf-8 # # This file is part of Hypothesis, which may be found at @@ -17,15 +19,8 @@ from __future__ import division, print_function, absolute_import -import os import sys - -import hypothesistooling as tools +import hashlib if __name__ == '__main__': - if os.environ.get('TRAVIS_SECURE_ENV_VARS', None) != 'true': - sys.exit(0) - - tools.decrypt_secrets() - - assert os.path.exists(tools.DEPLOY_KEY) + print(hashlib.sha1(sys.stdin.read().encode('utf-8')).hexdigest()[:10]) diff --git a/tooling/setup.py b/tooling/setup.py new file mode 100644 index 0000000000..52da61e093 --- /dev/null +++ b/tooling/setup.py @@ -0,0 +1,48 @@ +# coding=utf-8 +# +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis-python +# +# Most of this work is copyright (C) 2013-2018 David R. MacIver +# (david@drmaciver.com), but it contains contributions by others. See +# CONTRIBUTING.rst for a full list of people who may hold copyright, and +# consult the git log if you need to determine who owns an individual +# contribution. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. +# +# END HEADER + +from __future__ import division, print_function, absolute_import + +import os + +import setuptools + + +def local_file(name): + return os.path.relpath(os.path.join(os.path.dirname(__file__), name)) + + +SOURCE = local_file('src') +README = local_file('README.rst') + +setuptools.setup( + name='hypothesis-tooling', + # We don't actually ship this, it just has a setup.py for convenience. + version='0.0.0', + author='David R. MacIver', + author_email='david@drmaciver.com', + packages=setuptools.find_packages(SOURCE), + package_dir={'': SOURCE}, + url=( + 'https://github.com/HypothesisWorks/hypothesis-python/' + 'tree/master/tooling' + ), + license='MPL v2', + description='A library for property based testing', + python_requires='>=3.6', + long_description=open(README).read(), +) diff --git a/scripts/hypothesistooling.py b/tooling/src/hypothesistooling/__init__.py similarity index 94% rename from scripts/hypothesistooling.py rename to tooling/src/hypothesistooling/__init__.py index adf72aa61d..76ba20595d 100644 --- a/scripts/hypothesistooling.py +++ b/tooling/src/hypothesistooling/__init__.py @@ -40,13 +40,17 @@ def tags(): ROOT = subprocess.check_output([ - 'git', 'rev-parse', '--show-toplevel']).decode('ascii').strip() + 'git', '-C', os.path.dirname(__file__), 'rev-parse', '--show-toplevel', +]).decode('ascii').strip() HYPOTHESIS_PYTHON = os.path.join(ROOT, 'hypothesis-python') PYTHON_SRC = os.path.join(HYPOTHESIS_PYTHON, 'src') PYTHON_TESTS = os.path.join(HYPOTHESIS_PYTHON, 'tests') +PYUP_FILE = os.path.join(ROOT, '.pyup.yml') + + assert os.path.exists(PYTHON_SRC) @@ -204,8 +208,11 @@ def modified_files(): def all_files(): - return subprocess.check_output(['git', 'ls-files']).decode( - 'ascii').splitlines() + return [ + f for f in subprocess.check_output( + ['git', 'ls-files']).decode('ascii').splitlines() + if os.path.exists(f) + ] RELEASE_FILE = os.path.join(HYPOTHESIS_PYTHON, 'RELEASE.rst') @@ -370,9 +377,9 @@ def changed_files_from_master(): return files -def should_run_ci_task(task, is_pull_request): +def should_run_ci_task(task): """Given a task name, should we run this task?""" - if not is_pull_request: + if not IS_PULL_REQUEST: print('We only skip tests if the job is a pull request.') return True @@ -432,3 +439,16 @@ def decrypt_secrets(): assert os.path.exists(DEPLOY_KEY) assert os.path.exists(PYPIRC) os.chmod(DEPLOY_KEY, int('0600', 8)) + + +IS_TRAVIS_PULL_REQUEST = ( + os.environ.get('TRAVIS_EVENT_TYPE') == 'pull_request' +) + +IS_CIRCLE_PULL_REQUEST = ( + os.environ.get('CIRCLE_BRANCH') == 'master' and + os.environ.get('CI_PULL_REQUESTS', '') != '' +) + + +IS_PULL_REQUEST = IS_TRAVIS_PULL_REQUEST or IS_CIRCLE_PULL_REQUEST diff --git a/tooling/src/hypothesistooling/__main__.py b/tooling/src/hypothesistooling/__main__.py new file mode 100644 index 0000000000..4d8c824a5d --- /dev/null +++ b/tooling/src/hypothesistooling/__main__.py @@ -0,0 +1,541 @@ +# coding=utf-8 +# +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis-python +# +# Most of this work is copyright (C) 2013-2018 David R. MacIver +# (david@drmaciver.com), but it contains contributions by others. See +# CONTRIBUTING.rst for a full list of people who may hold copyright, and +# consult the git log if you need to determine who owns an individual +# contribution. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. +# +# END HEADER + +from __future__ import division, print_function, absolute_import + +import os +import sys +import random +import shutil +import subprocess +from glob import glob +from time import time, sleep +from datetime import datetime + +import yaml +from pyup.config import Config + +import hypothesistooling as tools +import hypothesistooling.installers as install +from hypothesistooling import fix_doctests as fd +from hypothesistooling.scripts import pip_tool + +TASKS = {} + + +def task(fn): + name = fn.__name__.replace('_', '-') + TASKS[name] = fn + return fn + + +@task +def lint(): + pip_tool( + 'flake8', + *[f for f in tools.all_files() if f.endswith('.py')], + '--config', os.path.join(tools.ROOT, '.flake8'), + ) + + +@task +def check_pyup_yml(): + with open(tools.PYUP_FILE, 'r') as i: + data = yaml.safe_load(i.read()) + config = Config() + config.update_config(data) + + if not config.is_valid_schedule(): + print('Schedule %r is invalid' % (config.schedule,)) + sys.exit(1) + + +DIST = os.path.join(tools.HYPOTHESIS_PYTHON, 'dist') +PENDING_STATUS = ('started', 'created') + + +@task +def deploy(): + os.chdir(tools.HYPOTHESIS_PYTHON) + + last_release = tools.latest_version() + + print('Current version: %s. Latest released version: %s' % ( + tools.__version__, last_release + )) + + HEAD = tools.hash_for_name('HEAD') + MASTER = tools.hash_for_name('origin/master') + print('Current head:', HEAD) + print('Current master:', MASTER) + + on_master = tools.is_ancestor(HEAD, MASTER) + has_release = tools.has_release() + + if has_release: + print('Updating changelog and version') + tools.update_for_pending_release() + + print('Building an sdist...') + + if os.path.exists(DIST): + shutil.rmtree(DIST) + + subprocess.check_output([ + sys.executable, 'setup.py', 'sdist', '--dist-dir', DIST, + ]) + + if not on_master: + print('Not deploying due to not being on master') + sys.exit(0) + + if not has_release: + print('Not deploying due to no release') + sys.exit(0) + + start_time = time() + + prev_pending = None + + # We time out after an hour, which is a stupidly long time and it should + # never actually take that long: A full Travis run only takes about 20-30 + # minutes! This is really just here as a guard in case something goes + # wrong and we're not paying attention so as to not be too mean to Travis.. + while time() <= start_time + 60 * 60: + jobs = tools.build_jobs() + + failed_jobs = [ + (k, v) + for k, vs in jobs.items() + if k not in PENDING_STATUS + ('passed',) + for v in vs + ] + + if failed_jobs: + print('Failing this due to failure of jobs %s' % ( + ', '.join('%s(%s)' % (s, j) for j, s in failed_jobs), + )) + sys.exit(1) + else: + pending = [j for s in PENDING_STATUS for j in jobs.get(s, ())] + try: + # This allows us to test the deploy job for a build locally. + pending.remove('deploy') + except ValueError: + pass + if pending: + still_pending = set(pending) + if prev_pending is None: + print('Waiting for the following jobs to complete:') + for p in sorted(still_pending): + print(' * %s' % (p,)) + print() + else: + completed = prev_pending - still_pending + if completed: + print('%s completed since last check.' % ( + ', '.join(sorted(completed)),)) + prev_pending = still_pending + naptime = 10.0 * (2 + random.random()) + print('Waiting %.2fs for %d more job%s to complete' % ( + naptime, len(pending), 's' if len(pending) > 1 else '',)) + sleep(naptime) + else: + break + else: + print("We've been waiting for an hour. That seems bad. Failing now.") + sys.exit(1) + + print('Looks good to release!') + + if os.environ.get('TRAVIS_SECURE_ENV_VARS', None) != 'true': + print("But we don't have the keys to do it") + sys.exit(0) + + print('Decrypting secrets') + tools.decrypt_secrets() + + print('Release seems good. Pushing to github now.') + + tools.create_tag_and_push() + + print('Now uploading to pypi.') + + subprocess.check_call([ + sys.executable, '-m', 'twine', 'upload', + '--config-file', tools.PYPIRC, + os.path.join(DIST, '*'), + ]) + + sys.exit(0) + + +@task +def check_release_file(): + if tools.has_source_changes(): + if not tools.has_release(): + print( + 'There are source changes but no RELEASE.rst. Please create ' + 'one to describe your changes.' + ) + sys.exit(1) + tools.parse_release_file() + + +@task +def check_shellcheck(): + install.ensure_shellcheck() + subprocess.check_call([install.SHELLCHECK] + [ + f for f in tools.all_files() + if f.endswith('.sh') + ]) + + +@task +def check_rst(): + rst = glob('*.rst') + glob('guides/*.rst') + docs = glob('hypothesis-python/docs/*.rst') + + pip_tool('rst-lint', *rst) + pip_tool('flake8', '--select=W191,W291,W292,W293,W391', *(rst + docs)) + + +@task +def check_secrets(): + if os.environ.get('TRAVIS_SECURE_ENV_VARS', None) != 'true': + sys.exit(0) + + tools.decrypt_secrets() + + assert os.path.exists(tools.DEPLOY_KEY) + + +CURRENT_YEAR = datetime.utcnow().year + + +HEADER = """ +# coding=utf-8 +# +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis-python +# +# Most of this work is copyright (C) 2013-%(year)s David R. MacIver +# (david@drmaciver.com), but it contains contributions by others. See +# CONTRIBUTING.rst for a full list of people who may hold copyright, and +# consult the git log if you need to determine who owns an individual +# contribution. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. +# +# END HEADER""".strip() % { + 'year': CURRENT_YEAR, +} + + +@task +def format(): + def should_format_file(path): + if os.path.basename(path) in ( + 'header.py', 'test_lambda_formatting.py' + ): + return False + if 'vendor' in path.split(os.path.sep): + return False + return path.endswith('.py') + + changed = tools.modified_files() + + format_all = os.environ.get('FORMAT_ALL', '').lower() == 'true' + if 'scripts/header.py' in changed: + # We've changed the header, so everything needs its header updated. + format_all = True + if 'requirements/tools.txt' in changed: + # We've changed the tools, which includes a lot of our formatting + # logic, so we need to rerun formatters. + format_all = True + + files = tools.all_files() if format_all else changed + + files_to_format = [f for f in sorted(files) if should_format_file(f)] + + for f in files_to_format: + print(f) + lines = [] + with open(f, encoding='utf-8') as o: + shebang = None + first = True + header_done = False + for l in o.readlines(): + if first: + first = False + if l[:2] == '#!': + shebang = l + continue + if 'END HEADER' in l and not header_done: + lines = [] + header_done = True + else: + lines.append(l) + source = ''.join(lines).strip() + with open(f, 'w', encoding='utf-8') as o: + if shebang is not None: + o.write(shebang) + o.write('\n') + o.write(HEADER) + if source: + o.write('\n\n') + o.write(source) + o.write('\n') + pip_tool( + 'isort', '-p', 'hypothesis', '-ls', '-m', '2', '-w', '75', '-a', + 'from __future__ import absolute_import, print_function, division', + *files_to_format, + ) + + pip_tool('pyformat', '-i', *files_to_format) + + +VALID_STARTS = ( + '# coding=utf-8', + '#!/usr/bin/env python', +) + + +@task +def check_format(): + format() + n = max(map(len, VALID_STARTS)) + bad = False + for f in tools.all_files(): + if not f.endswith('.py'): + continue + with open(f, 'r', encoding='utf-8') as i: + start = i.read(n) + if not any(start.startswith(s) for s in VALID_STARTS): + print( + '%s has incorrect start %r' % (f, start), file=sys.stderr) + bad = True + if bad: + sys.exit(1) + check_not_changed() + + +def check_not_changed(): + subprocess.check_call(['git', 'diff', '--exit-code']) + + +@task +def fix_doctests(): + fd.main() + + +@task +def compile_requirements(upgrade=False): + if upgrade: + extra = ['--upgrade'] + else: + extra = [] + + os.chdir(tools.ROOT) + + for f in glob(os.path.join('requirements', '*.in')): + base, _ = os.path.splitext(f) + pip_tool('pip-compile', *extra, f, '--output-file', base + '.txt') + + +@task +def upgrade_requirements(): + compile_requirements(upgrade=True) + + +@task +def check_requirements(): + compile_requirements() + check_not_changed() + + +def update_changelog_for_docs(): + if not tools.has_release(): + return + if tools.has_uncommitted_changes(tools.CHANGELOG_FILE): + print( + 'Cannot build documentation with uncommitted changes to ' + 'changelog and a pending release. Please commit your changes or ' + 'delete your release file.') + sys.exit(1) + tools.update_changelog_and_version() + + +@task +def documentation(): + os.chdir(tools.HYPOTHESIS_PYTHON) + try: + update_changelog_for_docs() + pip_tool( + 'sphinx-build', '-W', '-b', 'html', '-d', 'docs/_build/doctrees', + 'docs', 'docs/_build/html' + ) + finally: + subprocess.check_call([ + 'git', 'checkout', 'docs/changes.rst', 'src/hypothesis/version.py' + ]) + + +@task +def doctest(): + os.chdir(tools.HYPOTHESIS_PYTHON) + env = dict(os.environ) + env['PYTHONPATH'] = 'src' + + pip_tool( + 'sphinx-build', '-W', '-b', 'doctest', '-d', 'docs/_build/doctrees', + 'docs', 'docs/_build/html', env=env, + ) + + +def run_tox(task, version): + python = install.python_executable(version) + + # Create a version of the name that tox will pick up for the correct + # interpreter alias. + linked_version = os.path.basename(python) + ALIASES[version] + try: + os.symlink(python, linked_version) + except FileExistsError: + pass + + os.chdir(tools.HYPOTHESIS_PYTHON) + env = dict(os.environ) + python = install.python_executable(version) + + env['PATH'] = os.path.dirname(python) + ':' + env['PATH'] + print(env['PATH']) + + pip_tool('tox', '-e', task, env=env) + + +PY27 = '2.7.14' +PY34 = '3.4.8' +PY35 = '3.5.5' +PY36 = '3.6.5' +PYPY2 = 'pypy2.7-5.10.0' + + +@task +def install_core(): + install.python_executable(PY27) + install.python_executable(PY36) + + +ALIASES = { + PYPY2: 'pypy', +} + +for n in [PY27, PY34, PY35, PY36]: + major, minor, patch = n.split('.') + ALIASES[n] = 'python%s.%s' % (major, minor) + + +@task +def check_py27(): + run_tox('py27-full', PY27) + + +@task +def check_py34(): + run_tox('py34-full', PY34) + + +@task +def check_py35(): + run_tox('py35-full', PY35) + + +@task +def check_py36(): + run_tox('py36-full', PY36) + + +@task +def check_pypy(): + run_tox('pypy-full', PYPY2) + + +@task +def check_py27_typing(): + run_tox('py27typing', PY27) + + +@task +def check_pypy_with_tracer(): + run_tox('pypy-with-tracer', PYPY2) + + +def standard_tox_task(name): + TASKS['check-' + name] = lambda: run_tox(name, PY36) + + +standard_tox_task('nose') +standard_tox_task('pytest28') +standard_tox_task('faker070') +standard_tox_task('faker-latest') +standard_tox_task('django20') +standard_tox_task('django111') + +for n in [19, 20, 21, 22, 23]: + standard_tox_task('pandas%d' % (n,)) + +standard_tox_task('examples3') +standard_tox_task('coverage') +standard_tox_task('pure-tracer') + + +@task +def check_quality(): + run_tox('quality', PY36) + run_tox('quality2', PY27) + + +@task +def check_examples2(): + run_tox('examples2', PY27) + + +@task +def check_unicode(): + run_tox('unicode', PY27) + + +if __name__ == '__main__': + if 'SNAKEPIT' not in os.environ: + print( + 'This module should not be executed directly, but instead via ' + 'build.sh (which sets up its environment)' + ) + sys.exit(1) + + task_to_run = os.environ.get('TASK') + if task_to_run is None: + task_to_run = sys.argv[1] + if not tools.should_run_ci_task(task_to_run): + sys.exit(0) + try: + TASKS[task_to_run]() + except subprocess.CalledProcessError as e: + sys.exit(e.returncode) diff --git a/scripts/fix_doctests.py b/tooling/src/hypothesistooling/fix_doctests.py similarity index 96% rename from scripts/fix_doctests.py rename to tooling/src/hypothesistooling/fix_doctests.py index b1adb27ef6..3017475ec3 100644 --- a/scripts/fix_doctests.py +++ b/tooling/src/hypothesistooling/fix_doctests.py @@ -28,6 +28,9 @@ from distutils.version import StrictVersion import hypothesistooling as tools +from hypothesistooling.scripts import tool_path + +SPHINXBUILD = tool_path('sphinx-build') def dedent_lines(lines, force_newline=None): @@ -104,7 +107,7 @@ def __repr__(self): def get_doctest_output(): # Return a dict of filename: list of examples, sorted from last to first # so that replacing them in sequence works - command = run(['sphinx-build', '-b', 'doctest', 'docs', 'docs/_build'], + command = run([SPHINXBUILD, '-b', 'doctest', 'docs', 'docs/_build'], stdout=PIPE, stderr=PIPE, encoding='utf-8') output = [FailingExample(c) for c in command.stdout.split('*' * 70) if c.strip().startswith('File "')] @@ -127,7 +130,7 @@ def indent_like(lines, like): def main(): os.chdir(tools.ROOT) - version = run(['sphinx-build', '--version'], stdout=PIPE, + version = run([SPHINXBUILD, '--version'], stdout=PIPE, encoding='utf-8').stdout.lstrip('sphinx-build ') if StrictVersion(version) < '1.7': print('This script requires Sphinx 1.7 or later; got %s.\n' % version) diff --git a/tooling/src/hypothesistooling/installers.py b/tooling/src/hypothesistooling/installers.py new file mode 100644 index 0000000000..c13721c4cd --- /dev/null +++ b/tooling/src/hypothesistooling/installers.py @@ -0,0 +1,83 @@ +# coding=utf-8 +# +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis-python +# +# Most of this work is copyright (C) 2013-2018 David R. MacIver +# (david@drmaciver.com), but it contains contributions by others. See +# CONTRIBUTING.rst for a full list of people who may hold copyright, and +# consult the git log if you need to determine who owns an individual +# contribution. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. +# +# END HEADER + +"""Module for obtaining various versions of Python. + +Currently this is a thin shim around pyenv, but it would be nice to have +this work on Windows as well by using Anaconda (as our build already +does). +""" + +from __future__ import division, print_function, absolute_import + +import os +import subprocess + +import hypothesistooling.scripts as scripts + +HOME = os.environ['HOME'] + + +def __python_executable(version): + return os.path.join(scripts.SNAKEPIT, version, 'bin', 'python') + + +def python_executable(version): + ensure_python(version) + return __python_executable(version) + + +PYTHONS = set() + + +def ensure_python(version): + if version in PYTHONS: + return + scripts.run_script('ensure-python.sh', version) + target = __python_executable(version) + assert os.path.exists(target), target + PYTHONS.add(version) + + +STACK = os.path.join(HOME, '.local', 'bin', 'stack') +GHC = os.path.join(HOME, '.local', 'bin', 'ghc') +SHELLCHECK = os.path.join(HOME, '.local', 'bin', 'shellcheck') + + +def ensure_stack(): + if os.path.exists(STACK): + return + subprocess.check_call('mkdir -p ~/.local/bin', shell=True) + subprocess.check_call( + 'curl -L https://www.stackage.org/stack/linux-x86_64 ' + '| tar xz --wildcards --strip-components=1 -C $HOME' + "/.local/bin '*/stack'", shell=True + ) + + +def ensure_ghc(): + if os.path.exists(GHC): + return + ensure_stack() + subprocess.check_call([STACK, 'setup']) + + +def ensure_shellcheck(): + if os.path.exists(SHELLCHECK): + return + ensure_ghc() + subprocess.check_call([STACK, 'install', 'shellcheck']) diff --git a/tooling/src/hypothesistooling/scripts.py b/tooling/src/hypothesistooling/scripts.py new file mode 100644 index 0000000000..e0f364fe11 --- /dev/null +++ b/tooling/src/hypothesistooling/scripts.py @@ -0,0 +1,80 @@ +# coding=utf-8 +# +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis-python +# +# Most of this work is copyright (C) 2013-2018 David R. MacIver +# (david@drmaciver.com), but it contains contributions by others. See +# CONTRIBUTING.rst for a full list of people who may hold copyright, and +# consult the git log if you need to determine who owns an individual +# contribution. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. +# +# END HEADER + +from __future__ import division, print_function, absolute_import + +import os +import re +import sys +import shlex +import subprocess + +from hypothesistooling import ROOT + + +def print_command(command, args): + args = list(args) + ranges = [] + for i, v in enumerate(args): + if os.path.exists(v): + if not ranges or ranges[-1][-1] < i - 1: + ranges.append([i, i]) + elif ranges[-1][-1] + 1 == i: + ranges[-1][-1] += 1 + for i, j in ranges: + if j > i: + args[i] = '...' + for k in range(i + 1, j + 1): + args[k] = None + args = [v for v in args if v is not None] + print(command, *map(shlex.quote, args)) + + +def run_script(script, *args, **kwargs): + print_command(script, args) + return subprocess.check_call( + [os.path.join(SCRIPTS, script), *args], **kwargs + ) + + +SCRIPTS = os.path.join(ROOT, 'tooling', 'scripts') +COMMON = os.path.join(SCRIPTS, 'common.sh') + + +def __calc_script_variables(): + exports = re.compile(r"export ([A-Z_]+)(=|$)") + + with open(COMMON) as i: + common = i.read() + + for name, _ in exports.findall(common): + globals()[name] = os.environ[name] + + +__calc_script_variables() + + +def tool_path(name): + return os.path.join(os.path.dirname(sys.executable), name) + + +def pip_tool(name, *args, **kwargs): + print_command(name, args) + r = subprocess.call([tool_path(name), *args], **kwargs) + + if r != 0: + sys.exit(r)