From bff165fbd057823e2545b1a69275a6394d7fd5f3 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Wed, 7 Feb 2024 20:25:54 +0100 Subject: [PATCH] CI: chain requirements to ensure compatibility (#3112) *Issue #, if available:* atm, all install requirements are installed as stand-alone, which means if there is a version collision, `pip` does not care about it, and so for eventual version compatibility, a collision may not be discovered *Description of changes:* just install all needed requirements for the particular job at once so pip sees all required packages and finds an intersection among all :flags: By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. cc: @jaheba @kashif @lostella kind ling as it seems there are no revivers by default :chipmunk: **Please tag this pr with at least one of these labels to make our release process faster:** BREAKING, new feature, bug fix, other change, dev setup --- .github/workflows/docs.yml | 13 ++++++------ .github/workflows/doctests.yaml | 7 +++---- .github/workflows/flake8.yml | 1 - .github/workflows/mxnet_nightly.yml | 19 ++++++++--------- .github/workflows/style_type_checks.yml | 1 - .../workflows/test_release_unix_nightly.yml | 21 +++++++++---------- .../workflows/test_release_win32_nightly.yml | 19 +++++++---------- .github/workflows/tests-nixtla.yaml | 9 ++++---- .github/workflows/tests-nursery.yml | 19 +++++++++-------- .github/workflows/tests-prophet.yaml | 7 +++---- .github/workflows/tests-r.yml | 7 +++---- .github/workflows/tests-torch.yml | 11 +++++----- .github/workflows/tests-xgboost.yaml | 11 +++++----- .github/workflows/tests.yml | 7 +++---- .../requirements-rotbaum-extra-methods.txt | 2 +- 15 files changed, 70 insertions(+), 84 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9ee6ed851a..668c07bd4c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,13 +27,12 @@ jobs: - name: Install dependencies run: | - python -m pip install -U pip - pip install ".[arrow]" - pip install -r requirements/requirements-docs.txt - pip install -r requirements/requirements-mxnet.txt - pip install -r requirements/requirements-pytorch.txt - pip install -r requirements/requirements-rotbaum.txt - pip install -r requirements/requirements-extras-statsforecast.txt + pip install ".[arrow]" \ + -r requirements/requirements-docs.txt \ + -r requirements/requirements-mxnet.txt \ + -r requirements/requirements-pytorch.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-extras-statsforecast.txt - name: Install Pandoc run: | diff --git a/.github/workflows/doctests.yaml b/.github/workflows/doctests.yaml index a1cf4258f0..d29af1cf29 100644 --- a/.github/workflows/doctests.yaml +++ b/.github/workflows/doctests.yaml @@ -21,10 +21,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - python -m pip install -U pip - pip install -e . - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-arrow.txt + pip install -e . \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-arrow.txt - name: Test with pytest run: | pytest -n2 --doctest-modules \ diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 3828854614..36cda1470e 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -14,7 +14,6 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - python -m pip install -U pip pip install ruff pip install docformatter[tomli]==1.5.0 - name: Ruff (Flake8) diff --git a/.github/workflows/mxnet_nightly.yml b/.github/workflows/mxnet_nightly.yml index c30f3450af..beec24bf6a 100644 --- a/.github/workflows/mxnet_nightly.yml +++ b/.github/workflows/mxnet_nightly.yml @@ -24,17 +24,16 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install pip==20.2 + pip install pip==20.2 pip install --pre "mxnet<2" -f https://dist.mxnet.io/python - pip install torch==1.10.0+cpu --no-cache-dir -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install pytorch-lightning~=1.5 - pip install -e ".[shell]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-sagemaker-sdk.txt - pip install -r requirements/requirements-extras-m-competitions.txt - pip install -r requirements/requirements-rotbaum.txt - pip install -r requirements/requirements-extras-anomaly-evaluation.txt - pip install -r requirements/requirements-extras-autogluon.txt + pip install "torch==1.10.0" "pytorch-lightning~=1.5" -f https://download.pytorch.org/whl/cpu/torch_stable.html + pip install -e ".[shell]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-sagemaker-sdk.txt \ + -r requirements/requirements-extras-m-competitions.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-extras-anomaly-evaluation.txt \ + -r requirements/requirements-extras-autogluon.txt - name: Test with pytest run: | pytest -m 'not (gpu or serial)' --cov src/gluonts --cov-report=term --cov-report xml test diff --git a/.github/workflows/style_type_checks.yml b/.github/workflows/style_type_checks.yml index f8ffcc4ca8..8ad3c0fa68 100644 --- a/.github/workflows/style_type_checks.yml +++ b/.github/workflows/style_type_checks.yml @@ -17,7 +17,6 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - python -m pip install -U pip pip install . pip install click black mypy pip install types-python-dateutil diff --git a/.github/workflows/test_release_unix_nightly.yml b/.github/workflows/test_release_unix_nightly.yml index 3fa9307d44..b602bc069a 100644 --- a/.github/workflows/test_release_unix_nightly.yml +++ b/.github/workflows/test_release_unix_nightly.yml @@ -23,17 +23,16 @@ jobs: run: | git clone https://github.com/awslabs/gluonts --branch $(curl https://api.github.com/repos/awslabs/gluonts/releases/latest | grep tag_name | cut -d : -f 2,3 | tr -d \"\ | tr -d \,\ ) cd gluon-ts - python -m pip install pip==20.2 - pip install mxnet~=1.8.0 - pip install torch==1.10.0+cpu --no-cache-dir -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install pytorch-lightning~=1.5 - pip install -e ".[shell]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-sagemaker-sdk.txt - pip install -r requirements/requirements-extras-m-competitions.txt - pip install -r requirements/requirements-rotbaum.txt - pip install -r requirements/requirements-extras-anomaly-evaluation.txt - pip install -r requirements/requirements-extras-autogluon.txt + pip install pip==20.2 + pip install "mxnet~=1.8.0" "torch==1.10.0" "pytorch-lightning~=1.5" \ + -f https://download.pytorch.org/whl/cpu/torch_stable.html + pip install -e ".[shell]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-sagemaker-sdk.txt \ + -r requirements/requirements-extras-m-competitions.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-extras-anomaly-evaluation.txt \ + -r requirements/requirements-extras-autogluon.txt - name: Test with pytest run: | cd gluon-ts diff --git a/.github/workflows/test_release_win32_nightly.yml b/.github/workflows/test_release_win32_nightly.yml index 961e921287..35a416182a 100644 --- a/.github/workflows/test_release_win32_nightly.yml +++ b/.github/workflows/test_release_win32_nightly.yml @@ -25,17 +25,14 @@ jobs: $tmp=$tmp.psobject.properties.value.trim() git clone https://github.com/awslabs/gluonts --branch $tmp cd gluon-ts - python -m pip install -U pip - pip install mxnet~=1.7.0 - pip install torch - pip install pytorch-lightning~=1.5 - pip install -e ".[shell]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-sagemaker-sdk.txt - pip install -r requirements/requirements-extras-m-competitions.txt - pip install -r requirements/requirements-rotbaum.txt - pip install -r requirements/requirements-extras-anomaly-evaluation.txt - pip install -r requirements/requirements-extras-autogluon.txt + pip install "mxnet~=1.7.0" torch "pytorch-lightning~=1.5" + pip install -e ".[shell]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-sagemaker-sdk.txt \ + -r requirements/requirements-extras-m-competitions.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-extras-anomaly-evaluation.txt \ + -r requirements/requirements-extras-autogluon.txt shell: powershell - name: Test with pytest run: | diff --git a/.github/workflows/tests-nixtla.yaml b/.github/workflows/tests-nixtla.yaml index c9cca4cccf..86aefed138 100644 --- a/.github/workflows/tests-nixtla.yaml +++ b/.github/workflows/tests-nixtla.yaml @@ -21,11 +21,10 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - python -m pip install -U pip - pip install . - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-statsforecast.txt - pip install -r requirements/requirements-extras-hierarchicalforecast.txt + pip install . \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-statsforecast.txt \ + -r requirements/requirements-extras-hierarchicalforecast.txt - name: Test with pytest run: | pytest -n2 --doctest-modules test/ext/statsforecast diff --git a/.github/workflows/tests-nursery.yml b/.github/workflows/tests-nursery.yml index b719f4e8fb..5bca25ec1c 100644 --- a/.github/workflows/tests-nursery.yml +++ b/.github/workflows/tests-nursery.yml @@ -29,15 +29,16 @@ jobs: run: pip install mxnet~=1.7.0 - name: Install other dependencies run: | - python -m pip install -U pip - pip install torch==1.10.0+cpu --no-cache-dir -f https://download.pytorch.org/whl/cpu/torch_stable.html - pip install pytorch-lightning~=1.5 - pip install -e ".[shell]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-sagemaker-sdk.txt - pip install -r requirements/requirements-extras-m-competitions.txt - pip install -r requirements/requirements-extras-anomaly-evaluation.txt - pip install -r requirements/requirements-extras-autogluon.txt + pip install "torch==1.10.0" "pytorch-lightning~=1.5" \ + -f https://download.pytorch.org/whl/cpu/torch_stable.html + pip install + pip install -e ".[shell]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-sagemaker-sdk.txt \ + -r requirements/requirements-extras-m-competitions.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-extras-anomaly-evaluation.txt \ + -r requirements/requirements-extras-autogluon.txt - name: Test with pytest run: | pytest -n2 --doctest-modules test/nursery diff --git a/.github/workflows/tests-prophet.yaml b/.github/workflows/tests-prophet.yaml index bd2f33b620..c1d3bdc586 100644 --- a/.github/workflows/tests-prophet.yaml +++ b/.github/workflows/tests-prophet.yaml @@ -21,10 +21,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - python -m pip install -U pip - pip install . - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-prophet.txt + pip install . \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-prophet.txt - name: Test with pytest run: | pytest -n2 --doctest-modules test/ext/prophet diff --git a/.github/workflows/tests-r.yml b/.github/workflows/tests-r.yml index c7e3ccac9d..52c8ab6206 100644 --- a/.github/workflows/tests-r.yml +++ b/.github/workflows/tests-r.yml @@ -38,10 +38,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Python dependencies run: | - python -m pip install -U pip - pip install . - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-r.txt + pip install . \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-r.txt - name: Test with pytest run: | pytest -n2 test/ext/r_forecast diff --git a/.github/workflows/tests-torch.yml b/.github/workflows/tests-torch.yml index 08cff85e5f..3763f6f01f 100644 --- a/.github/workflows/tests-torch.yml +++ b/.github/workflows/tests-torch.yml @@ -21,12 +21,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install -U pip - pip install ".[arrow]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-m-competitions.txt - pip install -r requirements/requirements-pytorch.txt - pip install -r requirements/requirements-extras-cpflows.txt + pip install ".[arrow]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-m-competitions.txt \ + -r requirements/requirements-pytorch.txt \ + -r requirements/requirements-extras-cpflows.txt - name: Test with pytest run: | pytest -n2 --doctest-modules --ignore test/nursery test diff --git a/.github/workflows/tests-xgboost.yaml b/.github/workflows/tests-xgboost.yaml index a8a859dbb1..022b766cf6 100644 --- a/.github/workflows/tests-xgboost.yaml +++ b/.github/workflows/tests-xgboost.yaml @@ -21,12 +21,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - python -m pip install -U pip setuptools wheel - python -m pip install -U "Cython<3.0" - pip install . - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-rotbaum.txt - pip install -r requirements/requirements-rotbaum-extra-methods.txt + pip install "Cython<3.0" + pip install . \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-rotbaum.txt \ + -r requirements/requirements-rotbaum-extra-methods.txt - name: Test with pytest run: | pytest -n2 --timeout 120 --doctest-modules test/ext/rotbaum diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a398b822e6..7119f0d83d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,10 +21,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install other dependencies run: | - python -m pip install -U pip - pip install ".[mxnet,arrow,shell]" - pip install -r requirements/requirements-test.txt - pip install -r requirements/requirements-extras-m-competitions.txt + pip install ".[mxnet,arrow,shell]" \ + -r requirements/requirements-test.txt \ + -r requirements/requirements-extras-m-competitions.txt - name: Test with pytest run: | pytest -n2 --doctest-modules --ignore test/nursery test diff --git a/requirements/requirements-rotbaum-extra-methods.txt b/requirements/requirements-rotbaum-extra-methods.txt index b06879ed3f..15c5376d9c 100644 --- a/requirements/requirements-rotbaum-extra-methods.txt +++ b/requirements/requirements-rotbaum-extra-methods.txt @@ -1,2 +1,2 @@ lightgbm>=2.3.1 -scikit-garden>=0.1.2 +# scikit-garden @ https://github.com/scikit-garden/scikit-garden/archive/refs/heads/master.zip