From 6c124cb5f3e03b0c6b69cd7b5cffee5955df0f70 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 24 Mar 2021 17:41:43 +0100 Subject: [PATCH 1/7] draft --- .github/workflows/ci_test-conda.yml | 81 +++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/ci_test-conda.yml diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml new file mode 100644 index 00000000000..cc5e820e522 --- /dev/null +++ b/.github/workflows/ci_test-conda.yml @@ -0,0 +1,81 @@ +name: PyTorch & Conda + +# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows +on: # Trigger the workflow on push or pull request, but only for the master branch + push: + branches: [master, "release/*"] + pull_request: + branches: [master, "release/*"] + +jobs: + conda: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04] + python-version: [3.7] + pytorch-version: [1.3, 1.4, 1.5, 1.6, 1.7] + + # Timeout: https://stackoverflow.com/a/59076067/4521646 + timeout-minutes: 35 + steps: + - uses: actions/checkout@v2 + + - name: Setup PyTorch version + run: | + python -c "fname = 'environment.yml' ; req = open(fname).read().replace('torch>=1.3', 'torch=${{ matrix.pytorch-version }}') ; open(fname, 'w').write(req)" + cat environment.yml + + - name: Cache conda + uses: actions/cache@v2 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}-${{ hashFiles('environment.yml') }} + restore-keys: | + ${{ runner.os }}-conda-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}- + + # Add another cache for Pip as not all packages lives in Conda env + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}-${{ hashFiles('requirements/base.txt') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}- + + # https://docs.conda.io/projects/conda/en/4.6.0/_downloads/52a95608c49671267e40c689e0bc00ca/conda-cheatsheet.pdf + # https://gist.github.com/mwouts/9842452d020c08faf9e84a3bba38a66f + - name: Setup Miniconda + uses: goanpeca/setup-miniconda@v1.6.0 + with: + # auto-update-conda: true + auto-activate-base: false + # miniconda-version: 4.7.12 # This downloads a new conda, use the conda-version + conda-version: 4.7.12 + python-version: ${{ matrix.python-version }} + environment-file: environment.yml + activate-environment: lightning + use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! + + - name: Environment + run: | + conda info + conda list + pip install --requirement requirements/test.txt + pip list + + - name: Tests + run: | + # NOTE: run coverage on tests does not propagare faler status for Win, https://github.com/nedbat/coveragepy/issues/1003 + python -m pytest pytorch_lightning tests -v --durations=0 --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml + shell: bash + + - name: Upload pytest test results + uses: actions/upload-artifact@master + with: + name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }} + path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml + # Use always() to always run this step to publish test results when there are test failures + if: failure() From 7d76068b94c246aebf971cec346bd75c6eb512da Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 24 Mar 2021 17:56:11 +0100 Subject: [PATCH 2/7] update --- .github/workflows/ci_test-conda.yml | 57 +++++++++++------------ MANIFEST.in | 1 + azure-pipelines.yml | 2 +- integrations/__init__.py | 3 ++ integrations/lightning_models.py | 12 ++++- integrations/requirements.txt | 1 + requirements.dev.txt | 8 ++++ tests/classification/test_stat_scores.py | 10 ++-- torchmetrics/utilities/imports.py | 58 ++++++++++++++++++++++++ 9 files changed, 113 insertions(+), 39 deletions(-) create mode 100644 integrations/requirements.txt create mode 100644 requirements.dev.txt diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml index cc5e820e522..b828675fcee 100644 --- a/.github/workflows/ci_test-conda.yml +++ b/.github/workflows/ci_test-conda.yml @@ -9,73 +9,68 @@ on: # Trigger the workflow on push or pull request, but only for the master bra jobs: conda: - - runs-on: ${{ matrix.os }} + runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: - os: [ubuntu-20.04] python-version: [3.7] - pytorch-version: [1.3, 1.4, 1.5, 1.6, 1.7] + pytorch-version: [1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9] # Timeout: https://stackoverflow.com/a/59076067/4521646 timeout-minutes: 35 steps: - uses: actions/checkout@v2 - - name: Setup PyTorch version - run: | - python -c "fname = 'environment.yml' ; req = open(fname).read().replace('torch>=1.3', 'torch=${{ matrix.pytorch-version }}') ; open(fname, 'w').write(req)" - cat environment.yml - - name: Cache conda uses: actions/cache@v2 with: path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}-${{ hashFiles('environment.yml') }} - restore-keys: | - ${{ runner.os }}-conda-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}- + key: conda-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}-${{ hashFiles('environment.yml') }} + restore-keys: conda-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}- # Add another cache for Pip as not all packages lives in Conda env - name: Cache pip uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}-${{ hashFiles('requirements/base.txt') }} - restore-keys: | - ${{ runner.os }}-pip-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}- + key: pip-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}-${{ hashFiles('requirements/base.txt') }} + restore-keys: pip-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}- # https://docs.conda.io/projects/conda/en/4.6.0/_downloads/52a95608c49671267e40c689e0bc00ca/conda-cheatsheet.pdf # https://gist.github.com/mwouts/9842452d020c08faf9e84a3bba38a66f - name: Setup Miniconda - uses: goanpeca/setup-miniconda@v1.6.0 + uses: conda-incubator/setup-miniconda@v2 with: - # auto-update-conda: true - auto-activate-base: false - # miniconda-version: 4.7.12 # This downloads a new conda, use the conda-version - conda-version: 4.7.12 - python-version: ${{ matrix.python-version }} - environment-file: environment.yml - activate-environment: lightning - use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! + miniconda-version: "4.7.12" + python-version: ${{ matrix.python-version }} + channels: conda-forge,pytorch,pytorch-test,pytorch-nightly + channel-priority: true + auto-activate-base: true + # environment-file: ./environment.yml + use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! - - name: Environment + - name: Update Environment run: | conda info + conda install pytorch=${{ matrix.pytorch-version }} cpuonly conda list - pip install --requirement requirements/test.txt + pip --version + pip install --requirement requirements.txt --upgrade-strategy only-if-needed --quiet + pip install --requirement tests/requirements.txt --upgrade-strategy only-if-needed --quiet pip list + python -c "import torch; assert torch.__version__[:3] == '${{ matrix.pytorch-version }}', torch.__version__" + shell: bash -l {0} - - name: Tests + - name: Testing run: | # NOTE: run coverage on tests does not propagare faler status for Win, https://github.com/nedbat/coveragepy/issues/1003 - python -m pytest pytorch_lightning tests -v --durations=0 --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml - shell: bash + python -m pytest torchmetrics tests -v --durations=35 --junitxml=junit/test-conda-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}.xml + shell: bash -l {0} - name: Upload pytest test results uses: actions/upload-artifact@master with: - name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }} - path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml + name: test-conda-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}.xml + path: junit/test-conda-py${{ matrix.python-version }}-pt${{ matrix.pytorch-version }}.xml # Use always() to always run this step to publish test results when there are test failures if: failure() diff --git a/MANIFEST.in b/MANIFEST.in index d71a9ae3643..e475eaac01b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -38,3 +38,4 @@ prune notebook* prune temp* prune test* prune benchmark* +prune integration* diff --git a/azure-pipelines.yml b/azure-pipelines.yml index eb28a1f1dd0..ad47cc3a360 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -84,5 +84,5 @@ jobs: condition: succeededOrFailed() - bash: | - python -m pytest integrations --durations=25 + python -m pytest integrations -v --durations=25 displayName: 'Integrations' diff --git a/integrations/__init__.py b/integrations/__init__.py index e69de29bb2d..edd0d0f590b 100644 --- a/integrations/__init__.py +++ b/integrations/__init__.py @@ -0,0 +1,3 @@ +from torchmetrics.utilities.imports import _module_available + +_PL_AVAILABLE = _module_available('pytorch_lightning') diff --git a/integrations/lightning_models.py b/integrations/lightning_models.py index 8482d76f77e..9f6ba6674de 100644 --- a/integrations/lightning_models.py +++ b/integrations/lightning_models.py @@ -11,10 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from unittest.mock import MagicMock + import torch -from pytorch_lightning import LightningModule from torch.utils.data import Dataset +from integrations import _PL_AVAILABLE + +if _PL_AVAILABLE: + import pytorch_lightning as pl +else: + pl = MagicMock() + class RandomDictStringDataset(Dataset): @@ -42,7 +50,7 @@ def __len__(self): return self.len -class BoringModel(LightningModule): +class BoringModel(pl.LightningModule): def __init__(self): """ diff --git a/integrations/requirements.txt b/integrations/requirements.txt new file mode 100644 index 00000000000..5c2802a7f46 --- /dev/null +++ b/integrations/requirements.txt @@ -0,0 +1 @@ +pytorch-lightning>=1.0 diff --git a/requirements.dev.txt b/requirements.dev.txt new file mode 100644 index 00000000000..39a457f26b8 --- /dev/null +++ b/requirements.dev.txt @@ -0,0 +1,8 @@ +# use mandatory dependencies +-r requirements.txt + +# add the testing dependencies +-r tests/requirements.txt + +# add the integration dependencies +-r integrations/requirements.txt \ No newline at end of file diff --git a/tests/classification/test_stat_scores.py b/tests/classification/test_stat_scores.py index 47daa099d17..64954d9913b 100644 --- a/tests/classification/test_stat_scores.py +++ b/tests/classification/test_stat_scores.py @@ -21,7 +21,7 @@ from torch import Tensor, tensor from tests.classification.inputs import _input_binary, _input_binary_prob, _input_multiclass -from tests.classification.inputs import _input_multiclass_prob as _input_mccls_prob +from tests.classification.inputs import _input_multiclass_prob as _input_mcls_prob from tests.classification.inputs import _input_multidim_multiclass as _input_mdmc from tests.classification.inputs import _input_multidim_multiclass_prob as _input_mdmc_prob from tests.classification.inputs import _input_multilabel as _input_mcls @@ -103,8 +103,8 @@ def _sk_stat_scores_mdim_mcls(preds, target, reduce, mdmc_reduce, num_classes, i ["macro", None, None, _input_binary, None], ["micro", None, None, _input_mdmc_prob, None], ["micro", None, None, _input_binary_prob, 0], - ["micro", None, None, _input_mccls_prob, NUM_CLASSES], - ["micro", None, NUM_CLASSES, _input_mccls_prob, NUM_CLASSES], + ["micro", None, None, _input_mcls_prob, NUM_CLASSES], + ["micro", None, NUM_CLASSES, _input_mcls_prob, NUM_CLASSES], ], ) def test_wrong_params(reduce, mdmc_reduce, num_classes, inputs, ignore_index): @@ -140,8 +140,8 @@ def test_wrong_threshold(): (_input_mlb_prob.preds, _input_mlb_prob.target, _sk_stat_scores, None, NUM_CLASSES, None, None), (_input_mlb_prob.preds, _input_mlb_prob.target, _sk_stat_scores, None, NUM_CLASSES, None, 2), (_input_mcls.preds, _input_mcls.target, _sk_stat_scores, None, NUM_CLASSES, False, None), - (_input_mccls_prob.preds, _input_mccls_prob.target, _sk_stat_scores, None, NUM_CLASSES, None, None), - (_input_mccls_prob.preds, _input_mccls_prob.target, _sk_stat_scores, None, NUM_CLASSES, None, 2), + (_input_mcls_prob.preds, _input_mcls_prob.target, _sk_stat_scores, None, NUM_CLASSES, None, None), + (_input_mcls_prob.preds, _input_mcls_prob.target, _sk_stat_scores, None, NUM_CLASSES, None, 2), (_input_multiclass.preds, _input_multiclass.target, _sk_stat_scores, None, NUM_CLASSES, None, None), (_input_mdmc.preds, _input_mdmc.target, _sk_stat_scores_mdim_mcls, "samplewise", NUM_CLASSES, None, None), ( diff --git a/torchmetrics/utilities/imports.py b/torchmetrics/utilities/imports.py index 9aa15dc8e82..9dd52f07e74 100644 --- a/torchmetrics/utilities/imports.py +++ b/torchmetrics/utilities/imports.py @@ -1,6 +1,64 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from distutils.version import LooseVersion +from importlib import import_module +from importlib.util import find_spec import torch +from pkg_resources import DistributionNotFound + + +def _module_available(module_path: str) -> bool: + """ + Check if a path is available in your environment + + >>> _module_available('os') + True + >>> _module_available('bla.bla') + False + """ + try: + return find_spec(module_path) is not None + except AttributeError: + # Python 3.6 + return False + except ModuleNotFoundError: + # Python 3.7+ + return False + + +def _compare_version(package: str, op, version) -> bool: + """ + Compare package version with some requirements + + >>> import operator + >>> _compare_version("torch", operator.ge, "0.1") + True + """ + try: + pkg = import_module(package) + except (ModuleNotFoundError, DistributionNotFound): + return False + try: + pkg_version = LooseVersion(pkg.__version__) + except AttributeError: + return False + if not (hasattr(pkg_version, "vstring") and hasattr(pkg_version, "version")): + # this is mock by sphinx, so it shall return True ro generate all summaries + return True + return op(pkg_version, LooseVersion(version)) + _TORCH_LOWER_1_4 = LooseVersion(torch.__version__) < LooseVersion("1.4.0") _TORCH_LOWER_1_5 = LooseVersion(torch.__version__) < LooseVersion("1.5.0") From e1ea456efcbade402e96434a9abddbe7af66b5fd Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 25 Mar 2021 12:59:24 +0100 Subject: [PATCH 3/7] clean --- integrations/lightning_models.py | 11 ++--------- integrations/requirements.txt | 1 - requirements.dev.txt | 8 -------- 3 files changed, 2 insertions(+), 18 deletions(-) delete mode 100644 integrations/requirements.txt delete mode 100644 requirements.dev.txt diff --git a/integrations/lightning_models.py b/integrations/lightning_models.py index 9f6ba6674de..31287be76b1 100644 --- a/integrations/lightning_models.py +++ b/integrations/lightning_models.py @@ -11,18 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from unittest.mock import MagicMock import torch +from pytorch_lightning import LightningModule from torch.utils.data import Dataset -from integrations import _PL_AVAILABLE - -if _PL_AVAILABLE: - import pytorch_lightning as pl -else: - pl = MagicMock() - class RandomDictStringDataset(Dataset): @@ -50,7 +43,7 @@ def __len__(self): return self.len -class BoringModel(pl.LightningModule): +class BoringModel(LightningModule): def __init__(self): """ diff --git a/integrations/requirements.txt b/integrations/requirements.txt deleted file mode 100644 index 5c2802a7f46..00000000000 --- a/integrations/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pytorch-lightning>=1.0 diff --git a/requirements.dev.txt b/requirements.dev.txt deleted file mode 100644 index 39a457f26b8..00000000000 --- a/requirements.dev.txt +++ /dev/null @@ -1,8 +0,0 @@ -# use mandatory dependencies --r requirements.txt - -# add the testing dependencies --r tests/requirements.txt - -# add the integration dependencies --r integrations/requirements.txt \ No newline at end of file From 1125ec5549fa31d4bbbc35547e296e6ffe33c584 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 25 Mar 2021 13:39:36 +0100 Subject: [PATCH 4/7] ... --- .github/workflows/ci_test-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml index b828675fcee..2114a505616 100644 --- a/.github/workflows/ci_test-conda.yml +++ b/.github/workflows/ci_test-conda.yml @@ -56,7 +56,7 @@ jobs: conda list pip --version pip install --requirement requirements.txt --upgrade-strategy only-if-needed --quiet - pip install --requirement tests/requirements.txt --upgrade-strategy only-if-needed --quiet + pip install --requirement requirements/test.txt --upgrade-strategy only-if-needed --quiet pip list python -c "import torch; assert torch.__version__[:3] == '${{ matrix.pytorch-version }}', torch.__version__" shell: bash -l {0} From 59bff6572560111cd32678d56d3ba411b185ca51 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 25 Mar 2021 13:52:03 +0100 Subject: [PATCH 5/7] ... --- setup.cfg | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 714df5ba65b..7e2f95eb99a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,7 +21,10 @@ exclude_lines = [flake8] max-line-length = 120 -exclude = .tox,*.egg,build,temp +exclude = + *.egg + build + temp select = E,W,F doctests = True verbose = 2 From ea35449ce72be72aa34a54691c034ac96c8cead6 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 25 Mar 2021 14:48:08 +0100 Subject: [PATCH 6/7] ... --- tests/retrieval/test_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/retrieval/test_map.py b/tests/retrieval/test_map.py index 277c5ae08c4..17dfa607d27 100644 --- a/tests/retrieval/test_map.py +++ b/tests/retrieval/test_map.py @@ -86,7 +86,7 @@ def test_input_data(torch_class_metric: Metric) -> None: """Check PL metrics inputs are controlled correctly. """ device = 'cuda' if torch.cuda.is_available() else 'cpu' - length = random.randint(0, 20) + length = random.randint(1, 20) # check error when `query_without_relevant_docs='error'` is raised correctly indexes = torch.tensor([0] * length, device=device, dtype=torch.int64) From 042c191ba0197aa6e0801d79d189b6cf175b9f24 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 25 Mar 2021 20:17:11 +0100 Subject: [PATCH 7/7] revert --- tests/retrieval/test_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/retrieval/test_map.py b/tests/retrieval/test_map.py index 17dfa607d27..277c5ae08c4 100644 --- a/tests/retrieval/test_map.py +++ b/tests/retrieval/test_map.py @@ -86,7 +86,7 @@ def test_input_data(torch_class_metric: Metric) -> None: """Check PL metrics inputs are controlled correctly. """ device = 'cuda' if torch.cuda.is_available() else 'cpu' - length = random.randint(1, 20) + length = random.randint(0, 20) # check error when `query_without_relevant_docs='error'` is raised correctly indexes = torch.tensor([0] * length, device=device, dtype=torch.int64)