diff --git a/.azure/gpu-pipeline.yml b/.azure/gpu-pipeline.yml index bad2a9465fc..b5407cf3bff 100644 --- a/.azure/gpu-pipeline.yml +++ b/.azure/gpu-pipeline.yml @@ -21,9 +21,10 @@ jobs: strategy: matrix: 'PyTorch | old': - docker-image: 'pytorch/pytorch:1.9.0-cuda11.1-cudnn8-runtime' + # Torch does not have build wheels with old Torch versions for newer CUDA + docker-image: 'nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04' agent-pool: 'lit-rtx-3090' - torch-ver: '1.9.0' + torch-ver: '1.8.1' 'PyTorch | 1.X': docker-image: 'pytorch/pytorch:1.13.1-cuda11.6-cudnn8-runtime' agent-pool: 'lit-rtx-3090' @@ -60,10 +61,33 @@ jobs: steps: + - script: | + container_id=$(head -1 /proc/self/cgroup|cut -d/ -f3) + echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + /tmp/docker exec -t -u 0 $container_id \ + sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo" + echo "##vso[task.setvariable variable=CONTAINER_ID]$container_id" + displayName: 'Install Sudo in container (thanks Microsoft!)' + + - script: | + sudo apt-get update -q --fix-missing + sudo apt-get install -q -y --no-install-recommends \ + build-essential wget python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-distutils + sudo update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 + wget https://bootstrap.pypa.io/get-pip.py --progress=bar:force:noscroll --no-check-certificate + python get-pip.py + env: + PYTHON_VERSION: "3.8" + condition: startsWith(variables['docker-image'], 'nvidia/cuda:') + displayName: 'install python & pip' + - bash: | - echo "##vso[task.setvariable variable=CONTAINER_ID]$(head -1 /proc/self/cgroup|cut -d/ -f3)" echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)" - displayName: 'Set environment variables' + CUDA_version=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p') + CUDA_version_mm="${CUDA_version//'.'/''}" + echo "##vso[task.setvariable variable=CUDA_VERSION_MM]$CUDA_version_mm" + echo "##vso[task.setvariable variable=TORCH_URL]https://download.pytorch.org/whl/cu${CUDA_version_mm}/torch_stable.html" + displayName: 'set Env. vars' - bash: | whoami && id @@ -72,23 +96,18 @@ jobs: nvidia-smi echo $CUDA_VISIBLE_DEVICES echo $CONTAINER_ID + echo $TORCH_URL python --version pip --version pip cache dir pip list - python -c "import torch ; print(torch.cuda.get_arch_list())" displayName: 'Image info & NVIDIA' - - script: | - /tmp/docker exec -t -u 0 $CONTAINER_ID \ - sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - displayName: 'Install Sudo in container (thanks Microsoft!)' - - bash: | pip install -q packaging - python ./requirements/adjust-versions.py requirements.txt + python ./requirements/adjust-versions.py requirements.txt $(torch-ver) for fpath in `ls requirements/*.txt`; do - python ./requirements/adjust-versions.py $fpath + python ./requirements/adjust-versions.py $fpath $(torch-ver) done displayName: 'Adjust versions' @@ -97,7 +116,7 @@ jobs: sudo apt-get update -qq --fix-missing sudo apt-get install -y build-essential gcc g++ cmake ffmpeg git libsndfile1 unzip --no-install-recommends # pip install pip -U - pip install -e . -r ./requirements/devel.txt + pip install -e . -U -r ./requirements/devel.txt --find-links ${TORCH_URL} pip install mkl-service==2.4.0 # needed for the gpu multiprocessing pip list displayName: 'Install environment' @@ -105,7 +124,7 @@ jobs: - bash: | set -e python -c "from torch import __version__ as ver ; assert str(ver).split('+')[0] == '$(torch-ver)', f'PyTorch: {ver}'" - python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'GPU: {mgpu}'" + python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'found GPUs: {mgpu}'" displayName: 'Sanity check' - bash: | @@ -117,7 +136,7 @@ jobs: ls -lh $(TRANSFORMERS_CACHE) displayName: 'Show caches' - - bash: python -m pytest torchmetrics --timeout=180 --durations=50 + - bash: python -m pytest torchmetrics --timeout=240 --durations=50 env: SKIP_SLOW_DOCTEST: 1 workingDirectory: src @@ -131,7 +150,7 @@ jobs: workingDirectory: tests displayName: 'Pull testing data from S3' - - bash: python -m pytest unittests -v --cov=torchmetrics --timeout=180 --durations=500 + - bash: python -m pytest unittests -v --cov=torchmetrics --timeout=240 --durations=500 env: CUDA_LAUNCH_BLOCKING: 1 workingDirectory: tests @@ -162,11 +181,11 @@ jobs: python .github/assistant.py set-oldest-versions --req_files='["requirements/integrate.txt"]' cat requirements/integrate.txt displayName: 'Set oldest versions' - condition: eq(variables['torch-ver'], '1.9.0') + condition: eq(variables['torch-ver'], '1.8.1') - bash: | pip install -q -r requirements/integrate.txt - pip install -e . # force reinstall TM as it could be overwritten by integration's depenencies + pip install -e . # force reinstall TM as it could be overwritten by integration's dependencies pip list displayName: 'Install integrations' diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index c72ffbd177c..de8cc73fde8 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -26,3 +26,8 @@ jobs: actions-ref: main artifact-name: dist-packages-${{ github.sha }} import-name: "torchmetrics" + testing-matrix: | + { + "os": ["ubuntu-20.04", "macos-11", "windows-2022"], + "python-version": ["3.8", "3.10"] + } diff --git a/.github/workflows/ci-integrate.yml b/.github/workflows/ci-integrate.yml index 2b9d26a32f4..4f72707b435 100644 --- a/.github/workflows/ci-integrate.yml +++ b/.github/workflows/ci-integrate.yml @@ -28,10 +28,9 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, macOS-11, windows-2022] - python-version: ['3.7', '3.10'] + python-version: ['3.8', '3.10'] requires: ['oldest', 'latest'] exclude: - - {python-version: '3.7', requires: 'latest'} - {python-version: '3.10', requires: 'oldest'} - {python-version: '3.10', os: 'windows'} # todo: https://discuss.pytorch.org/t/numpy-is-not-available-error/146192 include: diff --git a/.github/workflows/ci-tests-full.yml b/.github/workflows/ci-tests-full.yml index c0f5e6cebef..64fb1be814c 100644 --- a/.github/workflows/ci-tests-full.yml +++ b/.github/workflows/ci-tests-full.yml @@ -44,11 +44,8 @@ jobs: - {os: 'windows-2022', python-version: '3.9', pytorch-version: '1.13.1'} - {os: 'windows-2022', python-version: '3.10', pytorch-version: '2.0.0'} # the oldest configurations - - {os: 'ubuntu-20.04', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'} - {os: 'ubuntu-20.04', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'} - - {os: 'macOS-11', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'} - {os: 'macOS-11', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'} - - {os: 'windows-2019', python-version: '3.7', pytorch-version: '1.8.1', requires: 'oldest'} - {os: 'windows-2019', python-version: '3.8', pytorch-version: '1.8.1', requires: 'oldest'} env: PYTORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html" diff --git a/CHANGELOG.md b/CHANGELOG.md index ea434702767..86b904a9b45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,7 +115,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed -- +- Support for python 3.7 ([#1640](https://github.com/Lightning-AI/metrics/pull/1640)) ### Fixed diff --git a/requirements.txt b/requirements.txt index cd478ca1cf4..2c3ecddf13d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package # in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment -numpy >=1.17.2 +numpy >=1.18.5 torch >=1.8.1, <=2.0.0 typing-extensions; python_version < '3.9' packaging # hotfix for utils, can be dropped with lit-utils >=0.5 diff --git a/requirements/adjust-versions.py b/requirements/adjust-versions.py index 300fb155314..1136b6b5d0b 100644 --- a/requirements/adjust-versions.py +++ b/requirements/adjust-versions.py @@ -82,7 +82,7 @@ def adjust(requires: str, torch_version: Optional[str] = None) -> str: requirements_path, torch_version = sys.argv[1:] else: requirements_path, torch_version = sys.argv[1], None - logging.info(f"requirements_path='{requirements_path}' with torch_version='{torch_version}'") + logging.info(f"requirements_path='{requirements_path}' with arg torch_version='{torch_version}'") with open(requirements_path) as fp: requirements = fp.read() diff --git a/requirements/classification_test.txt b/requirements/classification_test.txt index 2d87aec322b..ec0853412d2 100644 --- a/requirements/classification_test.txt +++ b/requirements/classification_test.txt @@ -1,6 +1,6 @@ # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package # in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment -pandas >1.0.0, <=1.5.3 +pandas >=1.4.0, <=1.5.3 netcal >1.0.0, <=1.3.5 # calibration_error fairlearn # group_fairness diff --git a/requirements/test.txt b/requirements/test.txt index 4b62db7b8f1..1cc2627049d 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -14,4 +14,4 @@ requests <=2.28.2 fire <=0.5.0 cloudpickle >1.3, <=2.2.1 -scikit-learn >1.0, <1.2.3 +scikit-learn >=1.1.1, <1.2.3 diff --git a/setup.py b/setup.py index 55c1ec455e6..01174cbac94 100755 --- a/setup.py +++ b/setup.py @@ -201,7 +201,7 @@ def _prepare_extras(skip_files: Tuple[str] = ("devel.txt", "doctest.txt", "integ include_package_data=True, zip_safe=False, keywords=["deep learning", "machine learning", "pytorch", "metrics", "AI"], - python_requires=">=3.7", + python_requires=">=3.8", setup_requires=[], install_requires=BASE_REQUIREMENTS, extras_require=_prepare_extras(), @@ -227,7 +227,6 @@ def _prepare_extras(skip_files: Tuple[str] = ("devel.txt", "doctest.txt", "integ # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tests/unittests/classification/test_precision_recall_curve.py b/tests/unittests/classification/test_precision_recall_curve.py index 91fa0a6fe73..bfc5f11e3ca 100644 --- a/tests/unittests/classification/test_precision_recall_curve.py +++ b/tests/unittests/classification/test_precision_recall_curve.py @@ -55,7 +55,6 @@ class TestBinaryPrecisionRecallCurve(MetricTester): @pytest.mark.parametrize("ignore_index", [None, -1, 0]) @pytest.mark.parametrize("ddp", [True, False]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_binary_precision_recall_curve(self, input, ddp, ignore_index): """Test class implementation of metric.""" preds, target = input @@ -74,7 +73,6 @@ def test_binary_precision_recall_curve(self, input, ddp, ignore_index): ) @pytest.mark.parametrize("ignore_index", [None, -1, 0]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_binary_precision_recall_curve_functional(self, input, ignore_index): """Test functional implementation of metric.""" preds, target = input @@ -182,7 +180,6 @@ class TestMulticlassPrecisionRecallCurve(MetricTester): @pytest.mark.parametrize("ignore_index", [None, -1]) @pytest.mark.parametrize("ddp", [True, False]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_multiclass_precision_recall_curve(self, input, ddp, ignore_index): """Test class implementation of metric.""" preds, target = input @@ -202,7 +199,6 @@ def test_multiclass_precision_recall_curve(self, input, ddp, ignore_index): ) @pytest.mark.parametrize("ignore_index", [None, -1]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_multiclass_precision_recall_curve_functional(self, input, ignore_index): """Test functional implementation of metric.""" preds, target = input @@ -304,7 +300,6 @@ class TestMultilabelPrecisionRecallCurve(MetricTester): @pytest.mark.parametrize("ignore_index", [None, -1, 0]) @pytest.mark.parametrize("ddp", [True, False]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_multilabel_precision_recall_curve(self, input, ddp, ignore_index): """Test class implementation of metric.""" preds, target = input @@ -324,7 +319,6 @@ def test_multilabel_precision_recall_curve(self, input, ddp, ignore_index): ) @pytest.mark.parametrize("ignore_index", [None, -1, 0]) - @pytest.mark.skipif(compare_version("sklearn", operator.lt, "1.1.0"), reason="Restricted to latest `sklearn`") def test_multilabel_precision_recall_curve_functional(self, input, ignore_index): """Test functional implementation of metric.""" preds, target = input diff --git a/tests/unittests/nominal/test_cramers.py b/tests/unittests/nominal/test_cramers.py index 474fbcd327b..c78a19fcd1c 100644 --- a/tests/unittests/nominal/test_cramers.py +++ b/tests/unittests/nominal/test_cramers.py @@ -90,9 +90,6 @@ def _dython_cramers_v_matrix(matrix, bias_correction, nan_strategy, nan_replace_ @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize( "preds, target", [ @@ -167,9 +164,6 @@ def test_cramers_v_differentiability(self, preds, target, bias_correction, nan_s @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize("bias_correction", [False, True]) @pytest.mark.parametrize(("nan_strategy", "nan_replace_value"), [("replace", 1.0), ("drop", None)]) def test_cramers_v_matrix(cramers_matrix_input, bias_correction, nan_strategy, nan_replace_value): diff --git a/tests/unittests/nominal/test_pearson.py b/tests/unittests/nominal/test_pearson.py index 6449f5f948c..b08ff675727 100644 --- a/tests/unittests/nominal/test_pearson.py +++ b/tests/unittests/nominal/test_pearson.py @@ -77,9 +77,6 @@ def _pd_pearsons_t_matrix(matrix): @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize( "preds, target", [ @@ -124,9 +121,6 @@ def test_pearsons_t_differentiability(self, preds, target): @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) def test_pearsons_contingency_coefficient_matrix(pearson_matrix_input): """Test matrix version of metric works as expected.""" tm_score = pearsons_contingency_coefficient_matrix(pearson_matrix_input) diff --git a/tests/unittests/nominal/test_theils_u.py b/tests/unittests/nominal/test_theils_u.py index 1526aca4b67..99aeed51959 100644 --- a/tests/unittests/nominal/test_theils_u.py +++ b/tests/unittests/nominal/test_theils_u.py @@ -88,9 +88,6 @@ def _dython_theils_u_matrix(matrix, nan_strategy, nan_replace_value): @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize( "preds, target", [ @@ -159,9 +156,6 @@ def test_theils_u_differentiability(self, preds, target, nan_strategy, nan_repla @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize(("nan_strategy", "nan_replace_value"), [("replace", 1.0), ("drop", None)]) def test_theils_u_matrix(theils_u_matrix_input, nan_strategy, nan_replace_value): """Test matrix version of metric works as expected.""" diff --git a/tests/unittests/nominal/test_tschuprows.py b/tests/unittests/nominal/test_tschuprows.py index 7bbcc61fbc3..4c2d704646e 100644 --- a/tests/unittests/nominal/test_tschuprows.py +++ b/tests/unittests/nominal/test_tschuprows.py @@ -74,9 +74,6 @@ def _pd_tschuprows_t_matrix(matrix): @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) @pytest.mark.parametrize( "preds, target", [ @@ -122,9 +119,6 @@ def test_tschuprows_t_differentiability(self, preds, target): @pytest.mark.skipif(compare_version("pandas", operator.lt, "1.3.2"), reason="`dython` package requires `pandas>=1.3.2`") -@pytest.mark.skipif( # TODO: testing on CUDA fails with pandas 1.3.5, and newer is not available for python 3.7 - torch.cuda.is_available(), reason="Tests fail on CUDA with the most up-to-date available pandas" -) def test_tschuprows_t_matrix(tschuprows_matrix_input): """Test matrix version of metric works as expected.""" tm_score = tschuprows_t_matrix(tschuprows_matrix_input, bias_correction=False)