From 4fd56fdeccf4550953b896f9af41c8b9b65b9ed8 Mon Sep 17 00:00:00 2001 From: semohr <39738318+semohr@users.noreply.github.com> Date: Thu, 10 Dec 2020 09:08:03 +0100 Subject: [PATCH 1/4] Small compatibility fix for master of Theano-PyMc (#4293) * Update theano-pymc version * Update infer_shape signatures * Updates for new double-underscore config settings Co-authored-by: Brandon T. Willard --- .github/workflows/arviz_compat.yml | 2 +- .github/workflows/pytest.yml | 2 +- .github/workflows/windows.yml | 2 +- conda-envs/environment-dev-py36.yml | 2 +- conda-envs/environment-dev-py37.yml | 2 +- conda-envs/environment-dev-py38.yml | 2 +- pymc3/__init__.py | 4 ++-- pymc3/distributions/multivariate.py | 2 +- pymc3/math.py | 4 ++-- pymc3/ode/ode.py | 2 +- requirements.txt | 2 +- scripts/test.sh | 2 +- 12 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/arviz_compat.yml b/.github/workflows/arviz_compat.yml index 4d1046bd221..1e899568e07 100644 --- a/.github/workflows/arviz_compat.yml +++ b/.github/workflows/arviz_compat.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} env: TEST_SUBSET: ${{ matrix.test-subset }} - THEANO_FLAGS: floatX=${{ matrix.floatx }},gcc.cxxflags='-march=native' + THEANO_FLAGS: floatX=${{ matrix.floatx }},gcc__cxxflags='-march=native' defaults: run: shell: bash -l {0} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index e1da3c3bb05..2c16e9b6bbb 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -59,7 +59,7 @@ jobs: runs-on: ${{ matrix.os }} env: TEST_SUBSET: ${{ matrix.test-subset }} - THEANO_FLAGS: floatX=${{ matrix.floatx }},gcc.cxxflags='-march=native' + THEANO_FLAGS: floatX=${{ matrix.floatx }},gcc__cxxflags='-march=native' defaults: run: shell: bash -l {0} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 83add158392..6291cc80946 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -17,7 +17,7 @@ jobs: runs-on: ${{ matrix.os }} env: TEST_SUBSET: ${{ matrix.test-subset }} - THEANO_FLAGS: floatX=${{ matrix.floatx }},gcc.cxxflags='-march=core2' + THEANO_FLAGS: floatX=${{ matrix.floatx }},gcc__cxxflags='-march=core2' defaults: run: shell: bash -l {0} diff --git a/conda-envs/environment-dev-py36.yml b/conda-envs/environment-dev-py36.yml index 77c11267a22..31f7c970985 100644 --- a/conda-envs/environment-dev-py36.yml +++ b/conda-envs/environment-dev-py36.yml @@ -5,7 +5,7 @@ channels: dependencies: - python=3.6 - arviz>=0.9 - - theano-pymc==1.0.11 + - theano-pymc==1.0.12 - numpy>=1.13 - scipy>=0.18 - pandas>=0.18 diff --git a/conda-envs/environment-dev-py37.yml b/conda-envs/environment-dev-py37.yml index 17bd6a90a11..7c8ad623321 100644 --- a/conda-envs/environment-dev-py37.yml +++ b/conda-envs/environment-dev-py37.yml @@ -5,7 +5,7 @@ channels: dependencies: - python=3.7 - arviz>=0.9 - - theano-pymc==1.0.11 + - theano-pymc==1.0.12 - numpy>=1.13 - scipy>=0.18 - pandas>=0.18 diff --git a/conda-envs/environment-dev-py38.yml b/conda-envs/environment-dev-py38.yml index 2eac820ced4..1d2176edcfd 100644 --- a/conda-envs/environment-dev-py38.yml +++ b/conda-envs/environment-dev-py38.yml @@ -5,7 +5,7 @@ channels: dependencies: - python=3.8 - arviz>=0.9 - - theano-pymc==1.0.11 + - theano-pymc==1.0.12 - numpy>=1.13 - scipy>=0.18 - pandas>=0.18 diff --git a/pymc3/__init__.py b/pymc3/__init__.py index 788526adc5b..23751b518ff 100644 --- a/pymc3/__init__.py +++ b/pymc3/__init__.py @@ -32,8 +32,8 @@ def __set_compiler_flags(): # Workarounds for Theano compiler problems on various platforms import theano - current = theano.config.gcc.cxxflags - theano.config.gcc.cxxflags = f"{current} -Wno-c++11-narrowing" + current = theano.config.gcc__cxxflags + theano.config.gcc__cxxflags = f"{current} -Wno-c++11-narrowing" __set_compiler_flags() diff --git a/pymc3/distributions/multivariate.py b/pymc3/distributions/multivariate.py index 02a9f61955a..59851b2e0c6 100755 --- a/pymc3/distributions/multivariate.py +++ b/pymc3/distributions/multivariate.py @@ -724,7 +724,7 @@ def perform(self, node, inputs, outputs): pm._log.exception("Failed to check if %s positive definite", x) raise - def infer_shape(self, node, shapes): + def infer_shape(self, fgraph, node, shapes): return [[]] def grad(self, inp, grads): diff --git a/pymc3/math.py b/pymc3/math.py index bba098678af..a8d50f03c61 100644 --- a/pymc3/math.py +++ b/pymc3/math.py @@ -363,7 +363,7 @@ def grad(self, inputs, gout): idx = tt.arange(gz.shape[-1]) return [gz[..., idx, idx]] - def infer_shape(self, nodes, shapes): + def infer_shape(self, fgraph, nodes, shapes): return [(shapes[0][0],) + (shapes[0][1],) * 2] @@ -422,7 +422,7 @@ def grad(self, inputs, gout): ] return [gout[0][slc] for slc in slices] - def infer_shape(self, nodes, shapes): + def infer_shape(self, fgraph, nodes, shapes): first, second = zip(*shapes) return [(tt.add(*first), tt.add(*second))] diff --git a/pymc3/ode/ode.py b/pymc3/ode/ode.py index 4c1c3a77f2c..e15f370ea8a 100644 --- a/pymc3/ode/ode.py +++ b/pymc3/ode/ode.py @@ -213,7 +213,7 @@ def perform(self, node, inputs_storage, output_storage): # simulate states and sensitivities in one forward pass output_storage[0][0], output_storage[1][0] = self._simulate(y0, theta) - def infer_shape(self, node, input_shapes): + def infer_shape(self, fgraph, node, input_shapes): s_y0, s_theta = input_shapes output_shapes = [(self.n_times, self.n_states), (self.n_times, self.n_states, self.n_p)] return output_shapes diff --git a/requirements.txt b/requirements.txt index f97594daf9c..ff947e26f83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ numpy>=1.13.0 pandas>=0.18.0 patsy>=0.5.1 scipy>=0.18.1 -theano-pymc==1.0.11 +theano-pymc==1.0.12 typing-extensions>=3.7.4 diff --git a/scripts/test.sh b/scripts/test.sh index 42b01507d8c..f9ae8111f7b 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -3,4 +3,4 @@ set -e _FLOATX=${FLOATX:=float64} -THEANO_FLAGS="floatX=${_FLOATX},gcc.cxxflags='-march=core2'" pytest -v --cov=pymc3 --cov-report=xml "$@" --cov-report term +THEANO_FLAGS="floatX=${_FLOATX},gcc__cxxflags='-march=core2'" pytest -v --cov=pymc3 --cov-report=xml "$@" --cov-report term From b953c40455ac7403d56325f187c659dcf35b4884 Mon Sep 17 00:00:00 2001 From: Sayam Kumar Date: Fri, 11 Dec 2020 11:47:15 +0530 Subject: [PATCH 2/4] Made sample_shape same across all contexts in draw_values (#4305) * Made sample_shape same across all contexts, thereby resolves #3758 * Pass the failing test * Worked on suggestions * Used to_tuple for size * Given a mention in release notes * Update RELEASE-NOTES.md Co-authored-by: Thomas Wiecki --- RELEASE-NOTES.md | 5 +++++ pymc3/distributions/distribution.py | 1 + pymc3/distributions/simulator.py | 7 ++++--- pymc3/tests/test_distributions_random.py | 4 ++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index e6c404520be..83b56aa4ed9 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,5 +1,10 @@ # Release Notes +## PyMC3 3.10.1 (on deck) + +### Maintenance +- Make `sample_shape` same across all contexts in `draw_values` (see [#4305](https://github.com/pymc-devs/pymc3/pull/4305)). + ## PyMC3 3.10.0 (7 December 2020) This is a major release with many exciting new features. The biggest change is that we now rely on our own fork of [Theano-PyMC](https://github.com/pymc-devs/Theano-PyMC). This is in line with our [big announcement about our commitment to PyMC3 and Theano](https://pymc-devs.medium.com/the-future-of-pymc3-or-theano-is-dead-long-live-theano-d8005f8a0e9b). diff --git a/pymc3/distributions/distribution.py b/pymc3/distributions/distribution.py index 408da9a3129..238395d1b5d 100644 --- a/pymc3/distributions/distribution.py +++ b/pymc3/distributions/distribution.py @@ -690,6 +690,7 @@ def draw_values(params, point=None, size=None): """ # The following check intercepts and redirects calls to # draw_values in the context of sample_posterior_predictive + size = to_tuple(size) ppc_sampler = vectorized_ppc.get(None) if ppc_sampler is not None: # this is being done inside new, vectorized sample_posterior_predictive diff --git a/pymc3/distributions/simulator.py b/pymc3/distributions/simulator.py index c64ca3f1cff..1277ec4c82c 100644 --- a/pymc3/distributions/simulator.py +++ b/pymc3/distributions/simulator.py @@ -18,7 +18,7 @@ from scipy.spatial import cKDTree -from pymc3.distributions.distribution import NoDistribution, draw_values +from pymc3.distributions.distribution import NoDistribution, draw_values, to_tuple __all__ = ["Simulator"] @@ -114,11 +114,12 @@ def random(self, point=None, size=None): ------- array """ + size = to_tuple(size) params = draw_values([*self.params], point=point, size=size) - if size is None: + if len(size) == 0: return self.function(*params) else: - return np.array([self.function(*params) for _ in range(size)]) + return np.array([self.function(*params) for _ in range(size[0])]) def _str_repr(self, name=None, dist=None, formatting="plain"): if dist is None: diff --git a/pymc3/tests/test_distributions_random.py b/pymc3/tests/test_distributions_random.py index 3f8f73ddd0b..5b60aec91a5 100644 --- a/pymc3/tests/test_distributions_random.py +++ b/pymc3/tests/test_distributions_random.py @@ -1664,6 +1664,10 @@ def test_issue_3758(self): for var in "abcd": assert not np.isnan(np.std(samples[var])) + for var in "bcd": + std = np.std(samples[var] - samples["a"]) + np.testing.assert_allclose(std, 1, rtol=1e-2) + def test_issue_3829(self): with pm.Model() as model: x = pm.MvNormal("x", mu=np.zeros(5), cov=np.eye(5), shape=(2, 5)) From eb4be99ca5b1b257386f1dcb6ef8fdef89d004d9 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Fri, 11 Dec 2020 13:55:58 +0000 Subject: [PATCH 3/4] Only pin theano-pymc in requirements.txt (#4322) * :hammer: add scripts to check theano-pymc pins, to generate requirements-dev from conda, to sort conda deps * :wrench: update conda envs and requirements-dev.txt * :pencil: fix typo * :memo: add docstrings * :memo: update pip-to-conda docstring * noop * :fire: remove pin theano checks, just use requirements.txt * local install in arviz compat * :memo: switch -> checkout, note beginner friendly label * :memo: link to beginner friendly issues * run generate pip deps from conda on all conda env files * :wrench: make sure all conda-envs files generate requirements-dev * :twisted_rightwards_arrows: reorder some tests * :art: remove trailing comma * Update scripts/generate_pip_deps_from_conda.py Co-authored-by: Thomas Wiecki --- .github/workflows/arviz_compat.yml | 14 ++- .github/workflows/pytest.yml | 14 +++ .github/workflows/windows.yml | 17 +++ .pre-commit-config.yaml | 14 +++ CONTRIBUTING.md | 6 +- conda-envs/environment-dev-py36.yml | 47 ++++----- conda-envs/environment-dev-py37.yml | 43 +++----- conda-envs/environment-dev-py38.yml | 43 +++----- requirements-dev.txt | 32 +++--- scripts/generate_pip_deps_from_conda.py | 131 ++++++++++++++++++++++++ scripts/sort_conda_envs.py | 21 ++++ 11 files changed, 280 insertions(+), 102 deletions(-) create mode 100755 scripts/generate_pip_deps_from_conda.py create mode 100644 scripts/sort_conda_envs.py diff --git a/.github/workflows/arviz_compat.yml b/.github/workflows/arviz_compat.yml index 1e899568e07..34b4106f005 100644 --- a/.github/workflows/arviz_compat.yml +++ b/.github/workflows/arviz_compat.yml @@ -33,6 +33,18 @@ jobs: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('conda-envs/environment-dev-py38.yml') }} + - name: Cache multiple paths + uses: actions/cache@v2 + env: + # Increase this value to reset cache if requirements.txt has not changed + CACHE_NUMBER: 0 + with: + path: | + ~/.cache/pip + $RUNNER_TOOL_CACHE/Python/* + ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-build-${{ matrix.python-version }}-${{ + hashFiles('requirements.txt') }} - uses: conda-incubator/setup-miniconda@v2 with: activate-environment: pymc3-dev-py38 @@ -47,7 +59,7 @@ jobs: - name: Install latest arviz run: | conda activate pymc3-dev-py38 - conda remove arviz -y + pip uninstall arviz -y pip install git+git://github.com/arviz-devs/arviz.git - name: Run tests run: | diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2c16e9b6bbb..905128ca0f9 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -21,6 +21,7 @@ jobs: --ignore=pymc3/tests/test_examples.py --ignore=pymc3/tests/test_gp.py --ignore=pymc3/tests/test_mixture.py + --ignore=pymc3/tests/test_ode.py --ignore=pymc3/tests/test_parallel_sampling.py --ignore=pymc3/tests/test_posteriors.py --ignore=pymc3/tests/test_quadpotential.py @@ -43,6 +44,7 @@ jobs: - | pymc3/tests/test_examples.py pymc3/tests/test_mixture.py + pymc3/tests/test_ode.py pymc3/tests/test_posteriors.py pymc3/tests/test_quadpotential.py - | @@ -74,6 +76,18 @@ jobs: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('conda-envs/environment-dev-py36.yml') }} + - name: Cache multiple paths + uses: actions/cache@v2 + env: + # Increase this value to reset cache if requirements.txt has not changed + CACHE_NUMBER: 0 + with: + path: | + ~/.cache/pip + $RUNNER_TOOL_CACHE/Python/* + ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-build-${{ matrix.python-version }}-${{ + hashFiles('requirements.txt') }} - uses: conda-incubator/setup-miniconda@v2 with: activate-environment: pymc3-dev-py36 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 6291cc80946..b5170fc876a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -32,12 +32,29 @@ jobs: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('conda-envs/environment-dev-py37.yml') }} + - name: Cache multiple paths + uses: actions/cache@v2 + env: + # Increase this value to reset cache if requirements.txt has not changed + CACHE_NUMBER: 0 + with: + path: | + ~/.cache/pip + $RUNNER_TOOL_CACHE/Python/* + ~\AppData\Local\pip\Cache + key: ${{ runner.os }}-build-${{ matrix.python-version }}-${{ + hashFiles('requirements.txt') }} - uses: conda-incubator/setup-miniconda@v2 with: activate-environment: pymc3-dev-py37 channel-priority: strict environment-file: conda-envs/environment-dev-py37.yml use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! + - name: Install-pymc3 + run: | + conda activate pymc3-dev-py37 + pip install -e . + python --version - run: | conda activate pymc3-dev-py37 python -m pytest -vv --cov=pymc3 --cov-report=xml --cov-report term --durations=50 $TEST_SUBSET diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c9cb1b73b41..25d215c14dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,7 @@ repos: - id: no-commit-to-branch args: [--branch, master] - id: requirements-txt-fixer + exclude: ^requirements-dev\.txt$ - id: trailing-whitespace - repo: https://github.com/nbQA-dev/nbQA rev: 0.5.4 @@ -61,6 +62,19 @@ repos: language: python name: Check no tests are ignored pass_filenames: false + - id: conda-env-sort + additional_dependencies: [pyyaml] + entry: python scripts/sort_conda_envs.py + files: ^conda-envs/ + language: python + name: Sort dependencies in conda envs + types: [yaml] + - id: pip-from-conda + additional_dependencies: [pyyaml] + entry: python scripts/generate_pip_deps_from_conda.py + files: ^conda-envs/ + language: python + name: Generate pip dependency from conda - id: no-relative-imports name: No relative imports entry: from \.[\.\w]* import diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 05173c1e975..2d56fc4e8a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,8 @@ We appreciate being notified of problems with the existing PyMC code. We prefer Please verify that your issue is not being currently addressed by other issues or pull requests by using the GitHub search tool to look for key words in the project issue tracker. +Filter on the ["beginner friendly"](https://github.com/pymc-devs/pymc3/issues?q=is%3Aopen+is%3Aissue+label%3A%22beginner+friendly%22) label for issues which are good for new contributors. + # Contributing code via pull requests While issue reporting is valuable, we strongly encourage users who are inclined to do so to submit patches for new or existing issues via pull requests. This is particularly the case for simple fixes, such as typos or tweaks to documentation, which do not require a heavy investment of time and attention. @@ -38,7 +40,7 @@ The preferred workflow for contributing to PyMC3 is to fork the [GitHub reposito 3. Create a ``feature`` branch to hold your development changes: ```bash - $ git switch -c my-feature + $ git checkout -b my-feature ``` Always use a ``feature`` branch. It's good practice to never routinely work on the ``master`` branch of any repository. @@ -147,4 +149,6 @@ Follow [TensorFlow's style guide](https://www.tensorflow.org/community/contribut For documentation strings, we *prefer* [numpy style](https://numpydoc.readthedocs.io/en/latest/format.html) to comply with the style that predominates in our upstream dependencies. +Finally, see the [PyMC3 Python Code Style](https://github.com/pymc-devs/pymc3/wiki/PyMC3-Python-Code-Style) and the [PyMC's Jupyter Notebook Style](https://github.com/pymc-devs/pymc3/wiki/PyMC's-Jupyter-Notebook-Style) guides. + #### This guide was derived from the [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/master/CONTRIBUTING.md) diff --git a/conda-envs/environment-dev-py36.yml b/conda-envs/environment-dev-py36.yml index 31f7c970985..6f6bf11a887 100644 --- a/conda-envs/environment-dev-py36.yml +++ b/conda-envs/environment-dev-py36.yml @@ -1,31 +1,22 @@ name: pymc3-dev-py36 channels: - - conda-forge - - defaults +- conda-forge +- defaults dependencies: - - python=3.6 - - arviz>=0.9 - - theano-pymc==1.0.12 - - numpy>=1.13 - - scipy>=0.18 - - pandas>=0.18 - - patsy>=0.5 - - fastprogress>=0.2 - - h5py>=2.7 - - typing-extensions>=3.7 - - python-graphviz - - ipython>=7.16 - - nbsphinx>=0.4 - - numpydoc>=0.9 - - pre-commit>=2.8.0 - - pytest-cov>=2.5 - - pytest>=3.0 - - recommonmark>=0.4 - - sphinx-autobuild>=0.7 - - sphinx>=1.5 - - watermark - - dataclasses # python_version < 3.7 - - contextvars # python_version < 3.7 - - mkl-service - - dill - - libblas=*=*mkl +- contextvars +- dataclasses +- h5py>=2.7 +- ipython>=7.16 +- libblas=*=*mkl +- mkl-service +- nbsphinx>=0.4 +- numpydoc>=0.9 +- pre-commit>=2.8.0 +- pytest-cov>=2.5 +- pytest>=3.0 +- python-graphviz +- python=3.6 +- recommonmark>=0.4 +- sphinx-autobuild>=0.7 +- sphinx>=1.5 +- watermark diff --git a/conda-envs/environment-dev-py37.yml b/conda-envs/environment-dev-py37.yml index 7c8ad623321..1a307dfd0f1 100644 --- a/conda-envs/environment-dev-py37.yml +++ b/conda-envs/environment-dev-py37.yml @@ -1,29 +1,20 @@ name: pymc3-dev-py37 channels: - - conda-forge - - defaults +- conda-forge +- defaults dependencies: - - python=3.7 - - arviz>=0.9 - - theano-pymc==1.0.12 - - numpy>=1.13 - - scipy>=0.18 - - pandas>=0.18 - - patsy>=0.5 - - fastprogress>=0.2 - - h5py>=2.7 - - typing-extensions>=3.7 - - python-graphviz - - ipython>=7.16 - - nbsphinx>=0.4 - - numpydoc>=0.9 - - pre-commit>=2.8.0 - - pytest-cov>=2.5 - - pytest>=3.0 - - recommonmark>=0.4 - - sphinx-autobuild>=0.7 - - sphinx>=1.5 - - watermark - - mkl-service - - dill - - libblas=*=*mkl +- h5py>=2.7 +- ipython>=7.16 +- libblas=*=*mkl +- mkl-service +- nbsphinx>=0.4 +- numpydoc>=0.9 +- pre-commit>=2.8.0 +- pytest-cov>=2.5 +- pytest>=3.0 +- python-graphviz +- python=3.7 +- recommonmark>=0.4 +- sphinx-autobuild>=0.7 +- sphinx>=1.5 +- watermark diff --git a/conda-envs/environment-dev-py38.yml b/conda-envs/environment-dev-py38.yml index 1d2176edcfd..24e9040cb31 100644 --- a/conda-envs/environment-dev-py38.yml +++ b/conda-envs/environment-dev-py38.yml @@ -1,29 +1,20 @@ name: pymc3-dev-py38 channels: - - conda-forge - - defaults +- conda-forge +- defaults dependencies: - - python=3.8 - - arviz>=0.9 - - theano-pymc==1.0.12 - - numpy>=1.13 - - scipy>=0.18 - - pandas>=0.18 - - patsy>=0.5 - - fastprogress>=0.2 - - h5py>=2.7 - - typing-extensions>=3.7 - - python-graphviz - - ipython>=7.16 - - nbsphinx>=0.4 - - numpydoc>=0.9 - - pre-commit>=2.8.0 - - pytest-cov>=2.5 - - pytest>=3.0 - - recommonmark>=0.4 - - sphinx-autobuild>=0.7 - - sphinx>=1.5 - - watermark - - mkl-service - - dill - - libblas=*=*mkl +- h5py>=2.7 +- ipython>=7.16 +- libblas=*=*mkl +- mkl-service +- nbsphinx>=0.4 +- numpydoc>=0.9 +- pre-commit>=2.8.0 +- pytest-cov>=2.5 +- pytest>=3.0 +- python-graphviz +- python=3.8 +- recommonmark>=0.4 +- sphinx-autobuild>=0.7 +- sphinx>=1.5 +- watermark diff --git a/requirements-dev.txt b/requirements-dev.txt index 1e127fe8b9e..1396c4ee183 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,22 +1,14 @@ -bokeh>=0.12.13 -coverage>=5.1 -dill -graphviz>=0.8.3 -ipython -nbsphinx>=0.4.2 -nose>=1.3.7 -nose-parameterized==0.6.0 -numpydoc>=0.9.1 -parameterized +# This file is auto-generated from by scripts/generate_pip_deps_from_conda.py, do not modify. +# See that file for comments about the need/usage of each dependency. + +h5py>=2.7 +ipython>=7.16 +nbsphinx>=0.4 +numpydoc>=0.9 pre-commit>=2.8.0 -pycodestyle>=2.3.1 -pyflakes>=1.5.0 -pylint>=1.7.4 -pytest>=3.0.7 -pytest-cov>=2.5.1 -pytest-xdist -recommonmark>=0.4.0 -seaborn>=0.8.1 -sphinx>=1.5.5 -sphinx-autobuild==0.7.1 +pytest-cov>=2.5 +pytest>=3.0 +recommonmark>=0.4 +sphinx-autobuild>=0.7 +sphinx>=1.5 watermark diff --git a/scripts/generate_pip_deps_from_conda.py b/scripts/generate_pip_deps_from_conda.py new file mode 100755 index 00000000000..04a764d57e3 --- /dev/null +++ b/scripts/generate_pip_deps_from_conda.py @@ -0,0 +1,131 @@ +# BSD 3-Clause License + +# Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team +# All rights reserved. + +# Copyright (c) 2011-2020, Open source contributors. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Check requirements-dev.txt has been generated from conda-envs/environment-dev-py3*.yml + +This is intended to be used as a pre-commit hook, see `.pre-commit-config.yaml`. +You can run it manually with `pre-commit run pip-from-conda --all`. +""" +import argparse +import re + +import yaml + +EXCLUDE = {"python", "libblas", "mkl-service", "python-graphviz"} +RENAME = {} + + +def conda_package_to_pip(package): + """ + Convert a conda package to its pip equivalent. + + In most cases they are the same, those are the exceptions: + - Packages that should be excluded (in `EXCLUDE`) + - Packages that should be renamed (in `RENAME`) + - A package requiring a specific version, in conda is defined with a single + equal (e.g. ``pandas=1.0``) and in pip with two (e.g. ``pandas==1.0``) + """ + package = re.sub("(?<=[^<>])=", "==", package).strip() + + for compare in ("<=", ">=", "=="): + if compare not in package: + continue + pkg, version = package.split(compare, maxsplit=1) + if pkg in EXCLUDE: + return + + if pkg in RENAME: + return "".join((RENAME[pkg], compare, version)) + + break + + if package in EXCLUDE: + return + + if package in RENAME: + return RENAME[package] + + return package + + +def main(conda_fname, pip_fname): + """ + Generate the pip dependencies file from the conda file, or compare that + they are synchronized (``compare=True``). + + Parameters + ---------- + conda_fname : str + Path to the conda file with dependencies (e.g. `environment.yml`). + pip_fname : str + Path to the pip file with dependencies (e.g. `requirements-dev.txt`). + compare : bool, default False + Whether to generate the pip file (``False``) or to compare if the + pip file has been generated with this script and the last version + of the conda file (``True``). + + Returns + ------- + bool + True if the comparison fails, False otherwise + """ + with open(conda_fname) as conda_fd: + deps = yaml.safe_load(conda_fd)["dependencies"] + + pip_deps = [] + for dep in deps: + if isinstance(dep, str): + conda_dep = conda_package_to_pip(dep) + if conda_dep: + pip_deps.append(conda_dep) + elif isinstance(dep, dict) and len(dep) == 1 and "pip" in dep: + pip_deps += dep["pip"] + else: + raise ValueError(f"Unexpected dependency {dep}") + + header = ( + f"# This file is auto-generated by scripts/generate_pip_deps_from_conda.py, " + "do not modify.\n# See that file for comments about the need/usage of each dependency.\n\n" + ) + pip_content = header + "\n".join(pip_deps) + "\n" + + with open(pip_fname, "w") as pip_fd: + pip_fd.write(pip_content) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("files", nargs="*") + args = parser.parse_args() + for file in args.files: + main(file, "requirements-dev.txt") diff --git a/scripts/sort_conda_envs.py b/scripts/sort_conda_envs.py new file mode 100644 index 00000000000..299892858e4 --- /dev/null +++ b/scripts/sort_conda_envs.py @@ -0,0 +1,21 @@ +""" +Sort dependencies in conda environment files. + +This is intended to be used as a pre-commit hook, see `.pre-commit-config.yaml`. +You can run it manually with `pre-commit run conda-env-sort --all`. +""" + +import argparse + +import yaml + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("files", nargs="*") + args = parser.parse_args() + for file_ in args.files: + with open(file_) as fd: + doc = yaml.safe_load(fd) + doc["dependencies"].sort() + with open(file_, "w") as fd: + yaml.dump(doc, fd, sort_keys=False) From 2a38198a03a03805503f25edfd2e1f2fcdaf5297 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Fri, 11 Dec 2020 14:26:00 +0000 Subject: [PATCH 4/4] Fix failing pip-from-conda (#4326) * :ambulance: fix pip-from-conda * Update scripts/generate_pip_deps_from_conda.py --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1396c4ee183..ec92f04f0de 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -# This file is auto-generated from by scripts/generate_pip_deps_from_conda.py, do not modify. +# This file is auto-generated by scripts/generate_pip_deps_from_conda.py, do not modify. # See that file for comments about the need/usage of each dependency. h5py>=2.7