From 8fc94a4a7ed87550c089cd314d608ddc4ab5afaa Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Tue, 21 Mar 2023 14:21:21 +0100 Subject: [PATCH 01/19] drop 3.7 --- .github/workflows/ci-integrate.yml | 3 +-- .github/workflows/ci-tests-full.yml | 3 --- setup.py | 3 +-- tests/unittests/nominal/test_cramers.py | 6 ------ tests/unittests/nominal/test_pearson.py | 6 ------ tests/unittests/nominal/test_theils_u.py | 6 ------ tests/unittests/nominal/test_tschuprows.py | 6 ------ 7 files changed, 2 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci-integrate.yml b/.github/workflows/ci-integrate.yml index 2b9d26a32f4..4f72707b435 100644 --- a/.github/workflows/ci-integrate.yml +++ b/.github/workflows/ci-integrate.yml @@ -28,10 +28,9 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, macOS-11, windows-2022] - python-version: ['3.7', '3.10'] + python-version: ['3.8', '3.10'] requires: ['oldest', 'latest'] exclude: - - {python-version: '3.7', requires: 'latest'} - {python-version: '3.10', requires: 'oldest'} - {python-version: '3.10', os: 'windows'} # todo: https://discuss.pytorch.org/t/numpy-is-not-available-error/146192 include: diff --git a/.github/workflows/ci-tests-full.yml b/.github/workflows/ci-tests-full.yml index 3f98c51f797..e36472ecd5a 100644 --- a/.github/workflows/ci-tests-full.yml +++ b/.github/workflows/ci-tests-full.yml @@ -44,11 +44,8 @@ jobs: - {os: 'windows-2022', python-version: '3.9', pytorch-version: '1.13.1'} - {os: 'windows-2022', python-version: '3.10', pytorch-version: '2.0.0'} # the oldest configurations - - {os: 'ubuntu-20.04', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'} - {os: 'ubuntu-20.04', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'} - - {os: 'macOS-11', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'} - {os: 'macOS-11', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'} - - {os: 'windows-2019', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'} - {os: 'windows-2019', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'} env: PYTORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" diff --git a/setup.py b/setup.py index 55c1ec455e6..01174cbac94 100755 --- a/setup.py +++ b/setup.py @@ -201,7 +201,7 @@ def _prepare_extras(skip_files: Tuple[str] = ("devel.txt", "doctest.txt", "integ include_package_data=True, zip_safe=False, keywords=["deep learning", "machine learning", "pytorch", "metrics", "AI"], - python_requires=">=3.7", + python_requires=">=3.8", setup_requires=[], install_requires=BASE_REQUIREMENTS, extras_require=_prepare_extras(), @@ -227,7 +227,6 @@ def _prepare_extras(skip_files: Tuple[str] = ("devel.txt", "doctest.txt", "integ # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tests/unittests/nominal/test_cramers.py b/tests/unittests/nominal/test_cramers.py index 474fbcd327b..c78a19fcd1c 100644 --- a/tests/unittests/nominal/test_cramers.py +++ b/tests/unittests/nominal/test_cramers.py @@ -90,9 +90,6 @@ def _dython_cramers_v_matrix(matrix, bias_correction, nan_strategy, nan_replace_ @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize( "preds, target", [ @@ -167,9 +164,6 @@ def test_cramers_v_differentiability(self, preds, target, bias_correction, nan_s @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize("bias_correction", [False, True]) @pytest.mark.parametrize(("nan_strategy", "nan_replace_value"), [("replace", 1.0), ("drop", None)]) def test_cramers_v_matrix(cramers_matrix_input, bias_correction, nan_strategy, nan_replace_value): diff --git a/tests/unittests/nominal/test_pearson.py b/tests/unittests/nominal/test_pearson.py index 6449f5f948c..b08ff675727 100644 --- a/tests/unittests/nominal/test_pearson.py +++ b/tests/unittests/nominal/test_pearson.py @@ -77,9 +77,6 @@ def _pd_pearsons_t_matrix(matrix): @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize( "preds, target", [ @@ -124,9 +121,6 @@ def test_pearsons_t_differentiability(self, preds, target): @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) def test_pearsons_contingency_coefficient_matrix(pearson_matrix_input): """Test matrix version of metric works as expected.""" tm_score = pearsons_contingency_coefficient_matrix(pearson_matrix_input) diff --git a/tests/unittests/nominal/test_theils_u.py b/tests/unittests/nominal/test_theils_u.py index 1526aca4b67..99aeed51959 100644 --- a/tests/unittests/nominal/test_theils_u.py +++ b/tests/unittests/nominal/test_theils_u.py @@ -88,9 +88,6 @@ def _dython_theils_u_matrix(matrix, nan_strategy, nan_replace_value): @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize( "preds, target", [ @@ -159,9 +156,6 @@ def test_theils_u_differentiability(self, preds, target, nan_strategy, nan_repla @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize(("nan_strategy", "nan_replace_value"), [("replace", 1.0), ("drop", None)]) def test_theils_u_matrix(theils_u_matrix_input, nan_strategy, nan_replace_value): """Test matrix version of metric works as expected.""" diff --git a/tests/unittests/nominal/test_tschuprows.py b/tests/unittests/nominal/test_tschuprows.py index 7bbcc61fbc3..4c2d704646e 100644 --- a/tests/unittests/nominal/test_tschuprows.py +++ b/tests/unittests/nominal/test_tschuprows.py @@ -74,9 +74,6 @@ def _pd_tschuprows_t_matrix(matrix): @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize( "preds, target", [ @@ -122,9 +119,6 @@ def test_tschuprows_t_differentiability(self, preds, target): @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) def test_tschuprows_t_matrix(tschuprows_matrix_input): """Test matrix version of metric works as expected.""" tm_score = tschuprows_t_matrix(tschuprows_matrix_input, bias_correction=False) From 3fe96cabc9666e0d4fdde5ad4bd03da5e6749c5e Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Tue, 21 Mar 2023 14:37:32 +0100 Subject: [PATCH 02/19] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40ba935bf4b..0369cef30f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,7 +101,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed -- +- Support for python 3.7 ([#1640](https://github.com/Lightning-AI/metrics/pull/1640)) ### Fixed From 35fd9690888669f1988cf7421462f513fa06d96e Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Tue, 21 Mar 2023 14:46:36 +0100 Subject: [PATCH 03/19] check package --- .github/workflows/ci-checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index e895cec8704..9422b2033f5 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -26,3 +26,4 @@ jobs: actions-ref: main artifact-name: dist-packages-${{ github.sha }} import-name: "torchmetrics" + testing-matrix: {"os": ["ubuntu-20.04", "macos-11", "windows-2022"], "python-version": ["3.8", "3.10"]} From 179f08dd693f04167970363744d84d242c623d2a Mon Sep 17 00:00:00 2001 From: Jirka Date: Tue, 21 Mar 2023 19:07:38 +0100 Subject: [PATCH 04/19] sklearn --- requirements/test.txt | 2 +- .../unittests/classification/test_precision_recall_curve.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/requirements/test.txt b/requirements/test.txt index 3248b0f7ff6..a55a0fac97f 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -14,4 +14,4 @@ requests <=2.28.2 fire <=0.5.0 cloudpickle >1.3, <=2.2.1 -scikit-learn >1.0, <1.2.2 +scikit-learn >=1.1.0, <1.2.2 diff --git a/tests/unittests/classification/test_precision_recall_curve.py b/tests/unittests/classification/test_precision_recall_curve.py index 91fa0a6fe73..bfc5f11e3ca 100644 --- a/tests/unittests/classification/test_precision_recall_curve.py +++ b/tests/unittests/classification/test_precision_recall_curve.py @@ -55,7 +55,6 @@ class TestBinaryPrecisionRecallCurve(MetricTester): @pytest.mark.parametrize("ignore_index", [None, -1, 0]) @pytest.mark.parametrize("ddp", [True, False]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_binary_precision_recall_curve(self, input, ddp, ignore_index): """Test class implementation of metric.""" preds, target = input @@ -74,7 +73,6 @@ def test_binary_precision_recall_curve(self, input, ddp, ignore_index): ) @pytest.mark.parametrize("ignore_index", [None, -1, 0]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_binary_precision_recall_curve_functional(self, input, ignore_index): """Test functional implementation of metric.""" preds, target = input @@ -182,7 +180,6 @@ class TestMulticlassPrecisionRecallCurve(MetricTester): @pytest.mark.parametrize("ignore_index", [None, -1]) @pytest.mark.parametrize("ddp", [True, False]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_multiclass_precision_recall_curve(self, input, ddp, ignore_index): """Test class implementation of metric.""" preds, target = input @@ -202,7 +199,6 @@ def test_multiclass_precision_recall_curve(self, input, ddp, ignore_index): ) @pytest.mark.parametrize("ignore_index", [None, -1]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_multiclass_precision_recall_curve_functional(self, input, ignore_index): """Test functional implementation of metric.""" preds, target = input @@ -304,7 +300,6 @@ class TestMultilabelPrecisionRecallCurve(MetricTester): @pytest.mark.parametrize("ignore_index", [None, -1, 0]) @pytest.mark.parametrize("ddp", [True, False]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_multilabel_precision_recall_curve(self, input, ddp, ignore_index): """Test class implementation of metric.""" preds, target = input @@ -324,7 +319,6 @@ def test_multilabel_precision_recall_curve(self, input, ddp, ignore_index): ) @pytest.mark.parametrize("ignore_index", [None, -1, 0]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_multilabel_precision_recall_curve_functional(self, input, ignore_index): """Test functional implementation of metric.""" preds, target = input From a0735e65e5da3e368f86664289a6ced4401ad794 Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Tue, 28 Mar 2023 16:58:18 +0200 Subject: [PATCH 05/19] try fix req --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cd478ca1cf4..d8b51281a92 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package # in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment -numpy >=1.17.2 +numpy >=1.17.3 torch >=1.8.1, <=2.0.0 typing-extensions; python_version < '3.9' packaging # hotfix for utils, can be dropped with lit-utils >=0.5 From c383b67e2199259b5bb552d508603526fb442cba Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Wed, 29 Mar 2023 08:21:16 +0200 Subject: [PATCH 06/19] up req --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index 64d9b02e72f..1cc2627049d 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -14,4 +14,4 @@ requests <=2.28.2 fire <=0.5.0 cloudpickle >1.3, <=2.2.1 -scikit-learn >=1.1.0, <1.2.3 +scikit-learn >=1.1.1, <1.2.3 From bca48806b0389bd084504e96a24c9ad145e5eabd Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Wed, 29 Mar 2023 08:45:03 +0200 Subject: [PATCH 07/19] pandas up --- requirements/classification_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/classification_test.txt b/requirements/classification_test.txt index 2d87aec322b..ec0853412d2 100644 --- a/requirements/classification_test.txt +++ b/requirements/classification_test.txt @@ -1,6 +1,6 @@ # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package # in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment -pandas >1.0.0, <=1.5.3 +pandas >=1.4.0, <=1.5.3 netcal >1.0.0, <=1.3.5 # calibration_error fairlearn # group_fairness From aad96a279ea77ec8133ca3b0826fbfd8508306ca Mon Sep 17 00:00:00 2001 From: SkafteNicki Date: Wed, 29 Mar 2023 08:58:00 +0200 Subject: [PATCH 08/19] up numpy --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d8b51281a92..2c3ecddf13d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package # in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment -numpy >=1.17.3 +numpy >=1.18.5 torch >=1.8.1, <=2.0.0 typing-extensions; python_version < '3.9' packaging # hotfix for utils, can be dropped with lit-utils >=0.5 From 56c73fc1c92e3172f46fe7cf00214fc05399c946 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 3 Apr 2023 14:03:34 +0200 Subject: [PATCH 09/19] update Azure --- .azure/gpu-pipeline.yml | 42 ++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index bad2a9465fc..ad3e0f6b39a 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -21,9 +21,9 @@ jobs: strategy: matrix: 'PyTorch | old': - docker-image: 'pytorch/pytorch:1.9.0-cuda11.1-cudnn8-runtime' + docker-image: 'nvidia/cuda:11.6.1-cudnn8-devel-ubuntu20.04' agent-pool: 'lit-rtx-3090' - torch-ver: '1.9.0' + torch-ver: '1.8.1' 'PyTorch | 1.X': docker-image: 'pytorch/pytorch:1.13.1-cuda11.6-cudnn8-runtime' agent-pool: 'lit-rtx-3090' @@ -60,10 +60,34 @@ jobs: steps: + - script: | + container_id=$(head -1 /proc/self/cgroup|cut -d/ -f3) + echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + /tmp/docker exec -t -u 0 $container_id \ + sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo" + echo "##vso[task.setvariable variable=CONTAINER_ID]$container_id" + displayName: 'Install Sudo in container (thanks Microsoft!)' + - bash: | - echo "##vso[task.setvariable variable=CONTAINER_ID]$(head -1 /proc/self/cgroup|cut -d/ -f3)" echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)" - displayName: 'Set environment variables' + CUDA_version=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p') + CUDA_version_mm="${CUDA_version//'.'/''}" + echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$CUDA_version_mm" + echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${CUDA_version_mm}/torch_stable.html" + displayName: 'set Env. vars' + + - script: | + sudo apt-get update -q --fix-missing + sudo apt-get install -q -y --no-install-recommends \ + build-essential wget python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-distutils + sudo update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 + wget https://bootstrap.pypa.io/get-pip.py --progress=bar:force:noscroll --no-check-certificate + python get-pip.py + pip install "torch==$(torch-ver)" -U --find-links ${TORCH_URL} + env: + PYTHON_VERSION: "3.8" + condition: startsWith(variables['docker-image'], 'nvidia/cuda:') + displayName: 'install python & pip' - bash: | whoami && id @@ -72,6 +96,7 @@ jobs: nvidia-smi echo $CUDA_VISIBLE_DEVICES echo $CONTAINER_ID + echo $TORCH_URL python --version pip --version pip cache dir @@ -79,11 +104,6 @@ jobs: python -c "import torch ; print(torch.cuda.get_arch_list())" displayName: 'Image info & NVIDIA' - - script: | - /tmp/docker exec -t -u 0 $CONTAINER_ID \ - sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - displayName: 'Install Sudo in container (thanks Microsoft!)' - - bash: | pip install -q packaging python ./requirements/adjust-versions.py requirements.txt @@ -97,7 +117,7 @@ jobs: sudo apt-get update -qq --fix-missing sudo apt-get install -y build-essential gcc g++ cmake ffmpeg git libsndfile1 unzip --no-install-recommends # pip install pip -U - pip install -e . -r ./requirements/devel.txt + pip install -e . -r ./requirements/devel.txt --find-links ${TORCH_URL} pip install mkl-service==2.4.0 # needed for the gpu multiprocessing pip list displayName: 'Install environment' @@ -166,7 +186,7 @@ jobs: - bash: | pip install -q -r requirements/integrate.txt - pip install -e . # force reinstall TM as it could be overwritten by integration's depenencies + pip install -e . # force reinstall TM as it could be overwritten by integration's dependencies pip list displayName: 'Install integrations' From d56191f3294f05ac29869f53eea6e562fd1a1ce4 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 3 Apr 2023 14:25:04 +0200 Subject: [PATCH 10/19] cuda vis --- .azure/gpu-pipeline.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index ad3e0f6b39a..0a319bbbd4d 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -68,14 +68,6 @@ jobs: echo "##vso[task.setvariable variable=CONTAINER_ID]$container_id" displayName: 'Install Sudo in container (thanks Microsoft!)' - - bash: | - echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)" - CUDA_version=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p') - CUDA_version_mm="${CUDA_version//'.'/''}" - echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$CUDA_version_mm" - echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${CUDA_version_mm}/torch_stable.html" - displayName: 'set Env. vars' - - script: | sudo apt-get update -q --fix-missing sudo apt-get install -q -y --no-install-recommends \ @@ -83,12 +75,19 @@ jobs: sudo update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 wget https://bootstrap.pypa.io/get-pip.py --progress=bar:force:noscroll --no-check-certificate python get-pip.py - pip install "torch==$(torch-ver)" -U --find-links ${TORCH_URL} env: PYTHON_VERSION: "3.8" condition: startsWith(variables['docker-image'], 'nvidia/cuda:') displayName: 'install python & pip' + - bash: | + echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)" + CUDA_version=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p') + CUDA_version_mm="${CUDA_version//'.'/''}" + echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$CUDA_version_mm" + echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${CUDA_version_mm}/torch_stable.html" + displayName: 'set Env. vars' + - bash: | whoami && id lspci | egrep 'VGA|3D' @@ -117,7 +116,7 @@ jobs: sudo apt-get update -qq --fix-missing sudo apt-get install -y build-essential gcc g++ cmake ffmpeg git libsndfile1 unzip --no-install-recommends # pip install pip -U - pip install -e . -r ./requirements/devel.txt --find-links ${TORCH_URL} + pip install -e . "torch==$(torch-ver)" -U -r ./requirements/devel.txt --find-links ${TORCH_URL} pip install mkl-service==2.4.0 # needed for the gpu multiprocessing pip list displayName: 'Install environment' @@ -125,7 +124,7 @@ jobs: - bash: | set -e python -c "from torch import __version__ as ver ; assert str(ver).split('+')[0] == '$(torch-ver)', f'PyTorch: {ver}'" - python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'GPU: {mgpu}'" + python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'found GPUs: {mgpu}'" displayName: 'Sanity check' - bash: | From 0efb4671eca0dad8bb54e2951642559972b54698 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 3 Apr 2023 14:29:21 +0200 Subject: [PATCH 11/19] ... --- .azure/gpu-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index 0a319bbbd4d..3641f53656f 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -100,7 +100,7 @@ jobs: pip --version pip cache dir pip list - python -c "import torch ; print(torch.cuda.get_arch_list())" + # python -c "import torch ; print(torch.cuda.get_arch_list())" displayName: 'Image info & NVIDIA' - bash: | From 6ae3734d7b13cea1d614cb506e11fc5ffcca25fe Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 3 Apr 2023 14:33:33 +0200 Subject: [PATCH 12/19] torch --- .azure/gpu-pipeline.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index 3641f53656f..7aa4ee5aeb2 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -88,6 +88,11 @@ jobs: echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${CUDA_version_mm}/torch_stable.html" displayName: 'set Env. vars' + - script: | + pip install "torch==$(torch-ver)" -U --find-links ${TORCH_URL} + condition: startsWith(variables['docker-image'], 'nvidia/cuda:') + displayName: 'install Torch' + - bash: | whoami && id lspci | egrep 'VGA|3D' @@ -100,7 +105,7 @@ jobs: pip --version pip cache dir pip list - # python -c "import torch ; print(torch.cuda.get_arch_list())" + python -c "import torch ; print(torch.cuda.get_arch_list())" displayName: 'Image info & NVIDIA' - bash: | @@ -116,7 +121,7 @@ jobs: sudo apt-get update -qq --fix-missing sudo apt-get install -y build-essential gcc g++ cmake ffmpeg git libsndfile1 unzip --no-install-recommends # pip install pip -U - pip install -e . "torch==$(torch-ver)" -U -r ./requirements/devel.txt --find-links ${TORCH_URL} + pip install -e . -U -r ./requirements/devel.txt --find-links ${TORCH_URL} pip install mkl-service==2.4.0 # needed for the gpu multiprocessing pip list displayName: 'Install environment' From a1199df1eacc2bb831b662c6e4914f00f3385ede Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 3 Apr 2023 14:45:23 +0200 Subject: [PATCH 13/19] 1.9.0 --- .azure/gpu-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index 7aa4ee5aeb2..5108a67664d 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -23,7 +23,7 @@ jobs: 'PyTorch | old': docker-image: 'nvidia/cuda:11.6.1-cudnn8-devel-ubuntu20.04' agent-pool: 'lit-rtx-3090' - torch-ver: '1.8.1' + torch-ver: '1.9.0' 'PyTorch | 1.X': docker-image: 'pytorch/pytorch:1.13.1-cuda11.6-cudnn8-runtime' agent-pool: 'lit-rtx-3090' From dc89359a50c8eb9d90299cf1001fa2024e28388f Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 3 Apr 2023 15:21:19 +0200 Subject: [PATCH 14/19] timeout=240 --- .azure/gpu-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index 5108a67664d..ff64b8e1f01 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -141,7 +141,7 @@ jobs: ls -lh $(TRANSFORMERS_CACHE) displayName: 'Show caches' - - bash: python -m pytest torchmetrics --timeout=180 --durations=50 + - bash: python -m pytest torchmetrics --timeout=240 --durations=50 env: SKIP_SLOW_DOCTEST: 1 workingDirectory: src @@ -155,7 +155,7 @@ jobs: workingDirectory: tests displayName: 'Pull testing data from S3' - - bash: python -m pytest unittests -v --cov=torchmetrics --timeout=180 --durations=500 + - bash: python -m pytest unittests -v --cov=torchmetrics --timeout=240 --durations=500 env: CUDA_LAUNCH_BLOCKING: 1 workingDirectory: tests From 3c969555bffe8fc1f2aa94ed6ce0491f8c41b21d Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 3 Apr 2023 16:39:39 +0200 Subject: [PATCH 15/19] adjust --- .azure/gpu-pipeline.yml | 10 ++-------- requirements/adjust-versions.py | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index ff64b8e1f01..8ebfe86401a 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -88,11 +88,6 @@ jobs: echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${CUDA_version_mm}/torch_stable.html" displayName: 'set Env. vars' - - script: | - pip install "torch==$(torch-ver)" -U --find-links ${TORCH_URL} - condition: startsWith(variables['docker-image'], 'nvidia/cuda:') - displayName: 'install Torch' - - bash: | whoami && id lspci | egrep 'VGA|3D' @@ -105,14 +100,13 @@ jobs: pip --version pip cache dir pip list - python -c "import torch ; print(torch.cuda.get_arch_list())" displayName: 'Image info & NVIDIA' - bash: | pip install -q packaging - python ./requirements/adjust-versions.py requirements.txt + python ./requirements/adjust-versions.py requirements.txt $(torch-ver) for fpath in `ls requirements/*.txt`; do - python ./requirements/adjust-versions.py $fpath + python ./requirements/adjust-versions.py $fpath $(torch-ver) done displayName: 'Adjust versions' diff --git a/requirements/adjust-versions.py b/requirements/adjust-versions.py index 300fb155314..1136b6b5d0b 100644 --- a/requirements/adjust-versions.py +++ b/requirements/adjust-versions.py @@ -82,7 +82,7 @@ def adjust(requires: str, torch_version: Optional[str] = None) -> str: requirements_path, torch_version = sys.argv[1:] else: requirements_path, torch_version = sys.argv[1], None - logging.info(f"requirements_path='{requirements_path}' with torch_version='{torch_version}'") + logging.info(f"requirements_path='{requirements_path}' with arg torch_version='{torch_version}'") with open(requirements_path) as fp: requirements = fp.read() From d232a62e1eed44118d36902346967add2761ce9b Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 3 Apr 2023 17:04:22 +0200 Subject: [PATCH 16/19] 11.1 --- .azure/gpu-pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index 8ebfe86401a..054e1631a0b 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -21,7 +21,8 @@ jobs: strategy: matrix: 'PyTorch | old': - docker-image: 'nvidia/cuda:11.6.1-cudnn8-devel-ubuntu20.04' + # Torch does not have build wheels with old Torch versions for newer CUDA + docker-image: 'nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04' agent-pool: 'lit-rtx-3090' torch-ver: '1.9.0' 'PyTorch | 1.X': From cde9e1ff59b706ff7b152f4f6c4a8f2c7269741c Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 3 Apr 2023 17:05:11 +0200 Subject: [PATCH 17/19] 1.8.1 --- .azure/gpu-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index 054e1631a0b..7b30f8a5a14 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -24,7 +24,7 @@ jobs: # Torch does not have build wheels with old Torch versions for newer CUDA docker-image: 'nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04' agent-pool: 'lit-rtx-3090' - torch-ver: '1.9.0' + torch-ver: '1.8.1' 'PyTorch | 1.X': docker-image: 'pytorch/pytorch:1.13.1-cuda11.6-cudnn8-runtime' agent-pool: 'lit-rtx-3090' From 7893c4bbfa1da591df67341b8530e13b161a7754 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 3 Apr 2023 17:59:15 +0200 Subject: [PATCH 18/19] skip --- .azure/gpu-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index 7b30f8a5a14..b5407cf3bff 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -181,7 +181,7 @@ jobs: python .github/assistant.py set-oldest-versions --req_files='["requirements/integrate.txt"]' cat requirements/integrate.txt displayName: 'Set oldest versions' - condition: eq(variables['torch-ver'], '1.9.0') + condition: eq(variables['torch-ver'], '1.8.1') - bash: | pip install -q -r requirements/integrate.txt From 0f2fcafefa04f68079b2efa1d68a95f199bbb7aa Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 3 Apr 2023 18:38:17 +0200 Subject: [PATCH 19/19] str --- .github/workflows/ci-checks.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index 48cfc5e36d4..de8cc73fde8 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -26,4 +26,8 @@ jobs: actions-ref: main artifact-name: dist-packages-${{ github.sha }} import-name: "torchmetrics" - testing-matrix: {"os": ["ubuntu-20.04", "macos-11", "windows-2022"], "python-version": ["3.8", "3.10"]} + testing-matrix: | + { + "os": ["ubuntu-20.04", "macos-11", "windows-2022"], + "python-version": ["3.8", "3.10"] + }