From 841d45ed030fdf3368959a143a5523195d4637b3 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Wed, 23 Dec 2020 10:31:10 +0000 Subject: [PATCH 1/7] test on oldest supported versions during end-to-end CI --- .github/workflows/arviz_compat.yml | 12 ++++++------ .github/workflows/pytest.yml | 12 ++++++------ .github/workflows/windows.yml | 12 ++++++------ .pre-commit-config.yaml | 4 ++-- conda-envs/environment-dev-py37.yml | 1 + requirements-dev.txt | 1 + scripts/sort_conda_envs.py | 29 +++++++++++++++++++---------- setup.py | 1 - 8 files changed, 41 insertions(+), 31 deletions(-) diff --git a/.github/workflows/arviz_compat.yml b/.github/workflows/arviz_compat.yml index 34b4106f00..2a56a1805a 100644 --- a/.github/workflows/arviz_compat.yml +++ b/.github/workflows/arviz_compat.yml @@ -27,12 +27,12 @@ jobs: - name: Cache conda uses: actions/cache@v1 env: - # Increase this value to reset cache if environment-dev-py38.yml has not changed + # Increase this value to reset cache if environment-dev-py39.yml has not changed CACHE_NUMBER: 0 with: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('conda-envs/environment-dev-py38.yml') }} + hashFiles('conda-envs/environment-dev-py39.yml') }} - name: Cache multiple paths uses: actions/cache@v2 env: @@ -47,18 +47,18 @@ jobs: hashFiles('requirements.txt') }} - uses: conda-incubator/setup-miniconda@v2 with: - activate-environment: pymc3-dev-py38 + activate-environment: pymc3-dev-py39 channel-priority: strict - environment-file: conda-envs/environment-dev-py38.yml + environment-file: conda-envs/environment-dev-py39.yml use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! - name: Install pymc3 run: | - conda activate pymc3-dev-py38 + conda activate pymc3-dev-py39 pip install -e . python --version - name: Install latest arviz run: | - conda activate pymc3-dev-py38 + conda activate pymc3-dev-py39 pip uninstall arviz -y pip install git+git://github.com/arviz-devs/arviz.git - name: Run tests diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index a4f7eced28..1b33e899d3 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -70,12 +70,12 @@ jobs: - name: Cache conda uses: actions/cache@v1 env: - # Increase this value to reset cache if environment-dev-py39.yml has not changed + # Increase this value to reset cache if environment-dev-py37.yml has not changed CACHE_NUMBER: 0 with: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('conda-envs/environment-dev-py39.yml') }} + hashFiles('conda-envs/environment-dev-py37.yml') }} - name: Cache multiple paths uses: actions/cache@v2 env: @@ -90,18 +90,18 @@ jobs: hashFiles('requirements.txt') }} - uses: conda-incubator/setup-miniconda@v2 with: - activate-environment: pymc3-dev-py39 + activate-environment: pymc3-dev-py37 channel-priority: strict - environment-file: conda-envs/environment-dev-py39.yml + environment-file: conda-envs/environment-dev-py37.yml use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! - name: Install-pymc3 run: | - conda activate pymc3-dev-py39 + conda activate pymc3-dev-py37 pip install -e . python --version - name: Run tests run: | - conda activate pymc3-dev-py39 + conda activate pymc3-dev-py37 python -m pytest -vv --cov=pymc3 --cov-report=xml --cov-report term --durations=50 $TEST_SUBSET - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index b5170fc876..8a81e97b21 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -26,12 +26,12 @@ jobs: - name: Cache conda uses: actions/cache@v1 env: - # Increase this value to reset cache if conda-envs/environment-dev-py37.yml has not changed + # Increase this value to reset cache if conda-envs/environment-dev-py38.yml has not changed CACHE_NUMBER: 0 with: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('conda-envs/environment-dev-py37.yml') }} + hashFiles('conda-envs/environment-dev-py38.yml') }} - name: Cache multiple paths uses: actions/cache@v2 env: @@ -46,15 +46,15 @@ jobs: hashFiles('requirements.txt') }} - uses: conda-incubator/setup-miniconda@v2 with: - activate-environment: pymc3-dev-py37 + activate-environment: pymc3-dev-py38 channel-priority: strict - environment-file: conda-envs/environment-dev-py37.yml + environment-file: conda-envs/environment-dev-py38.yml use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! - name: Install-pymc3 run: | - conda activate pymc3-dev-py37 + conda activate pymc3-dev-py38 pip install -e . python --version - run: | - conda activate pymc3-dev-py37 + conda activate pymc3-dev-py38 python -m pytest -vv --cov=pymc3 --cov-report=xml --cov-report term --durations=50 $TEST_SUBSET diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a21968f41..d5818f59d6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,7 +42,7 @@ repos: name: Check no tests are ignored pass_filenames: false - id: conda-env-sort - additional_dependencies: [pyyaml] + additional_dependencies: [ruamel.yaml] entry: python scripts/sort_conda_envs.py files: ^conda-envs/ language: python @@ -51,7 +51,7 @@ repos: - id: pip-from-conda additional_dependencies: [pyyaml] entry: python scripts/generate_pip_deps_from_conda.py - files: ^conda-envs/ + files: ^conda-envs/environment-dev-py37\.yml$ language: python name: Generate pip dependency from conda - id: no-relative-imports diff --git a/conda-envs/environment-dev-py37.yml b/conda-envs/environment-dev-py37.yml index 1a307dfd0f..725fa7abf6 100644 --- a/conda-envs/environment-dev-py37.yml +++ b/conda-envs/environment-dev-py37.yml @@ -8,6 +8,7 @@ dependencies: - libblas=*=*mkl - mkl-service - nbsphinx>=0.4 +- numpy=1.13.0 - numpydoc>=0.9 - pre-commit>=2.8.0 - pytest-cov>=2.5 diff --git a/requirements-dev.txt b/requirements-dev.txt index ec92f04f0d..0ebf709c6d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,6 +4,7 @@ h5py>=2.7 ipython>=7.16 nbsphinx>=0.4 +numpy==1.13.0 numpydoc>=0.9 pre-commit>=2.8.0 pytest-cov>=2.5 diff --git a/scripts/sort_conda_envs.py b/scripts/sort_conda_envs.py index 299892858e..4ee240de6e 100644 --- a/scripts/sort_conda_envs.py +++ b/scripts/sort_conda_envs.py @@ -7,15 +7,24 @@ import argparse -import yaml +from pathlib import Path +from typing import Optional, Sequence -if __name__ == "__main__": +import ruamel.yaml + +yaml = ruamel.yaml.YAML() + + +def main(argv: Optional[Sequence[str]] = None) -> None: + """Sort dependencies in conda environment files.""" parser = argparse.ArgumentParser() - parser.add_argument("files", nargs="*") - args = parser.parse_args() - for file_ in args.files: - with open(file_) as fd: - doc = yaml.safe_load(fd) - doc["dependencies"].sort() - with open(file_, "w") as fd: - yaml.dump(doc, fd, sort_keys=False) + parser.add_argument("paths", nargs="*", type=Path) + args = parser.parse_args(argv) + for path in args.paths: + doc = yaml.load(path) + doc["dependencies"].sort() + yaml.dump(doc, path) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 114d774fdc..9b8091ba1b 100755 --- a/setup.py +++ b/setup.py @@ -85,5 +85,4 @@ def get_version(): python_requires=">=3.7", install_requires=install_reqs, tests_require=test_reqs, - test_suite="nose.collector", ) From 2d8063377c99a7c3b2987f0ebbf1cfe2452a3402 Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Thu, 31 Dec 2020 11:00:43 +0000 Subject: [PATCH 2/7] less strict pin --- conda-envs/environment-dev-py37.yml | 2 +- requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda-envs/environment-dev-py37.yml b/conda-envs/environment-dev-py37.yml index 725fa7abf6..2a3b114bca 100644 --- a/conda-envs/environment-dev-py37.yml +++ b/conda-envs/environment-dev-py37.yml @@ -8,7 +8,7 @@ dependencies: - libblas=*=*mkl - mkl-service - nbsphinx>=0.4 -- numpy=1.13.0 +- numpy=1.13 - numpydoc>=0.9 - pre-commit>=2.8.0 - pytest-cov>=2.5 diff --git a/requirements-dev.txt b/requirements-dev.txt index 0ebf709c6d..b8234a6e6d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ h5py>=2.7 ipython>=7.16 nbsphinx>=0.4 -numpy==1.13.0 +numpy==1.13 numpydoc>=0.9 pre-commit>=2.8.0 pytest-cov>=2.5 From 5dfa4874b0da15153e3c5a193c1f7d207284171e Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Thu, 31 Dec 2020 11:38:57 +0000 Subject: [PATCH 3/7] try 1.15 pin --- conda-envs/environment-dev-py37.yml | 2 +- requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda-envs/environment-dev-py37.yml b/conda-envs/environment-dev-py37.yml index 2a3b114bca..b47eb94d99 100644 --- a/conda-envs/environment-dev-py37.yml +++ b/conda-envs/environment-dev-py37.yml @@ -8,7 +8,7 @@ dependencies: - libblas=*=*mkl - mkl-service - nbsphinx>=0.4 -- numpy=1.13 +- numpy=1.15 - numpydoc>=0.9 - pre-commit>=2.8.0 - pytest-cov>=2.5 diff --git a/requirements-dev.txt b/requirements-dev.txt index b8234a6e6d..c663cb4957 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ h5py>=2.7 ipython>=7.16 nbsphinx>=0.4 -numpy==1.13 +numpy==1.15 numpydoc>=0.9 pre-commit>=2.8.0 pytest-cov>=2.5 From 04ed835f65ba8eeb25125ae3191b9336a707373e Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Thu, 31 Dec 2020 12:12:36 +0000 Subject: [PATCH 4/7] try pinning pandas and scipy too --- conda-envs/environment-dev-py37.yml | 2 ++ requirements-dev.txt | 2 ++ requirements.txt | 6 +++--- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/conda-envs/environment-dev-py37.yml b/conda-envs/environment-dev-py37.yml index b47eb94d99..c944732928 100644 --- a/conda-envs/environment-dev-py37.yml +++ b/conda-envs/environment-dev-py37.yml @@ -10,12 +10,14 @@ dependencies: - nbsphinx>=0.4 - numpy=1.15 - numpydoc>=0.9 +- pandas=0.24 - pre-commit>=2.8.0 - pytest-cov>=2.5 - pytest>=3.0 - python-graphviz - python=3.7 - recommonmark>=0.4 +- scipy=1.2 - sphinx-autobuild>=0.7 - sphinx>=1.5 - watermark diff --git a/requirements-dev.txt b/requirements-dev.txt index c663cb4957..341cf5857e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,10 +6,12 @@ ipython>=7.16 nbsphinx>=0.4 numpy==1.15 numpydoc>=0.9 +pandas==0.24 pre-commit>=2.8.0 pytest-cov>=2.5 pytest>=3.0 recommonmark>=0.4 +scipy==1.2 sphinx-autobuild>=0.7 sphinx>=1.5 watermark diff --git a/requirements.txt b/requirements.txt index 1e29e212ea..4a5c22b092 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ arviz>=0.9.0 dill fastprogress>=0.2.0 -numpy>=1.13.0 -pandas>=0.18.0 +numpy>=1.15.0 +pandas>=0.24.0 patsy>=0.5.1 -scipy>=0.18.1 +scipy>=1.2.0 theano-pymc==1.0.12 typing-extensions>=3.7.4 From 5fbc4c497d0602071dbb13b4078973201b97161d Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Thu, 31 Dec 2020 14:01:02 +0000 Subject: [PATCH 5/7] only test betabinom if scipy modern enough --- .codecov.yml | 2 +- .github/workflows/arviz_compat.yml | 1 + pymc3/tests/test_distributions.py | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.codecov.yml b/.codecov.yml index e7fe4624f7..6b3f1f0830 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,7 +1,7 @@ codecov: require_ci_to_pass: no notify: - after_n_builds: 9 + after_n_builds: 16 coverage: precision: 2 diff --git a/.github/workflows/arviz_compat.yml b/.github/workflows/arviz_compat.yml index 2a56a1805a..2bbf076205 100644 --- a/.github/workflows/arviz_compat.yml +++ b/.github/workflows/arviz_compat.yml @@ -12,6 +12,7 @@ jobs: os: [ubuntu-latest, macos-latest] floatx: [float64] test-subset: + - pymc3/tests/test_distributions.py - pymc3/tests/test_distributions_random.py - pymc3/tests/test_sampling.py fail-fast: false diff --git a/pymc3/tests/test_distributions.py b/pymc3/tests/test_distributions.py index 9a21e00c15..64ebcd2886 100644 --- a/pymc3/tests/test_distributions.py +++ b/pymc3/tests/test_distributions.py @@ -25,6 +25,8 @@ from numpy import array, exp, inf, log from numpy.testing import assert_allclose, assert_almost_equal, assert_equal +from packaging.version import parse +from scipy import __version__ as scipy_version from scipy import integrate from scipy.special import erf, logit @@ -97,6 +99,8 @@ from pymc3.theanof import floatX from pymc3.vartypes import continuous_types +SCIPY_VERSION = parse(scipy_version) + def get_lkj_cases(): """ @@ -1134,6 +1138,9 @@ def test_binomial(self): # Too lazy to propagate decimal parameter through the whole chain of deps @pytest.mark.xfail(condition=(theano.config.floatX == "float32"), reason="Fails on float32") + @pytest.mark.xfail( + condition=(SCIPY_VERSION < parse("1.4.0")), reason="betabinom is new in Scipy 1.4.0" + ) def test_beta_binomial(self): self.checkd( BetaBinomial, From d939eef1eee491333cac5c07457ec3c17d17626a Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Tue, 5 Jan 2021 12:18:26 +0000 Subject: [PATCH 6/7] revert sort_conda changes (will do in separate PR) --- .pre-commit-config.yaml | 4 ++-- scripts/sort_conda_envs.py | 29 ++++++++++------------------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b93869891e..c4b750894e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,7 +42,7 @@ repos: name: Check no tests are ignored pass_filenames: false - id: conda-env-sort - additional_dependencies: [ruamel.yaml] + additional_dependencies: [pyyaml] entry: python scripts/sort_conda_envs.py files: ^conda-envs/ language: python @@ -51,7 +51,7 @@ repos: - id: pip-from-conda additional_dependencies: [pyyaml] entry: python scripts/generate_pip_deps_from_conda.py - files: ^conda-envs/environment-dev-py37\.yml$ + files: ^conda-envs/ language: python name: Generate pip dependency from conda - id: no-relative-imports diff --git a/scripts/sort_conda_envs.py b/scripts/sort_conda_envs.py index 4ee240de6e..299892858e 100644 --- a/scripts/sort_conda_envs.py +++ b/scripts/sort_conda_envs.py @@ -7,24 +7,15 @@ import argparse -from pathlib import Path -from typing import Optional, Sequence - -import ruamel.yaml - -yaml = ruamel.yaml.YAML() - - -def main(argv: Optional[Sequence[str]] = None) -> None: - """Sort dependencies in conda environment files.""" - parser = argparse.ArgumentParser() - parser.add_argument("paths", nargs="*", type=Path) - args = parser.parse_args(argv) - for path in args.paths: - doc = yaml.load(path) - doc["dependencies"].sort() - yaml.dump(doc, path) - +import yaml if __name__ == "__main__": - main() + parser = argparse.ArgumentParser() + parser.add_argument("files", nargs="*") + args = parser.parse_args() + for file_ in args.files: + with open(file_) as fd: + doc = yaml.safe_load(fd) + doc["dependencies"].sort() + with open(file_, "w") as fd: + yaml.dump(doc, fd, sort_keys=False) From 96e2882fb094866bbdd3a55304e6130786f0601f Mon Sep 17 00:00:00 2001 From: Marco Gorelli Date: Tue, 5 Jan 2021 13:37:43 +0000 Subject: [PATCH 7/7] generate requirements-dev from py37 env --- .pre-commit-config.yaml | 2 +- requirements-dev.txt | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c4b750894e..468fe167e3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: - id: conda-env-sort additional_dependencies: [pyyaml] entry: python scripts/sort_conda_envs.py - files: ^conda-envs/ + files: ^conda-envs/environment-dev-py37\.yml$ language: python name: Sort dependencies in conda envs types: [yaml] diff --git a/requirements-dev.txt b/requirements-dev.txt index 341cf5857e..ec92f04f0d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,14 +4,11 @@ h5py>=2.7 ipython>=7.16 nbsphinx>=0.4 -numpy==1.15 numpydoc>=0.9 -pandas==0.24 pre-commit>=2.8.0 pytest-cov>=2.5 pytest>=3.0 recommonmark>=0.4 -scipy==1.2 sphinx-autobuild>=0.7 sphinx>=1.5 watermark