From 8fd18a4967cb75239f9f3ad868dec412a7565e1a Mon Sep 17 00:00:00 2001 From: Kaushik Bokka Date: Sun, 7 Mar 2021 02:35:17 +0530 Subject: [PATCH 01/10] fix torch._six PT 1.8 comp issue --- .gitignore | 6 ++++++ pl_bolts/datamodules/async_dataloader.py | 3 ++- pl_bolts/datasets/imagenet_dataset.py | 4 +++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 741916a9c3..89998fba46 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ *.py[cod] *$py.class +data/ # C extensions *.so @@ -163,3 +164,8 @@ tests/test-reports/ # MLflow mlruns/ + + +# Downloaded Data through Scripts +VOCdevkit/ +*.tar diff --git a/pl_bolts/datamodules/async_dataloader.py b/pl_bolts/datamodules/async_dataloader.py index 8d7a26adc7..a29c4b905c 100644 --- a/pl_bolts/datamodules/async_dataloader.py +++ b/pl_bolts/datamodules/async_dataloader.py @@ -1,10 +1,11 @@ +import collections.abc as container_abcs import re from queue import Queue from threading import Thread from typing import Any, Optional, Union import torch -from torch._six import container_abcs, string_classes +from torch._six import string_classes from torch.utils.data import DataLoader, Dataset diff --git a/pl_bolts/datasets/imagenet_dataset.py b/pl_bolts/datasets/imagenet_dataset.py index 7488487c93..607af8cbf7 100644 --- a/pl_bolts/datasets/imagenet_dataset.py +++ b/pl_bolts/datasets/imagenet_dataset.py @@ -2,6 +2,7 @@ import hashlib import os import shutil +import sys import tarfile import tempfile import zipfile @@ -9,7 +10,6 @@ import numpy as np import torch -from torch._six import PY3 from pl_bolts.utils import _TORCHVISION_AVAILABLE from pl_bolts.utils.warnings import warn_missing_pkg @@ -246,6 +246,8 @@ def extract_archive(from_path, to_path=None, remove_finished=False): if to_path is None: to_path = os.path.dirname(from_path) + PY3 = sys.version_info[0] == 3 + if _is_tar(from_path): with tarfile.open(from_path, 'r') as tar: tar.extractall(path=to_path) From 9adb6e74f2479bacf02e11cdd4e1a701df89eeb8 Mon Sep 17 00:00:00 2001 From: Kaushik Bokka Date: Sun, 7 Mar 2021 02:58:18 +0530 Subject: [PATCH 02/10] update torch vision --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4d3da28520..d5e1b90c11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -torch>=1.6 +torch==1.8 pytorch-lightning>=1.1.1, <1.2 \ No newline at end of file From 8a3b7e579a916133b8cff2a860d3b91b003ff28f Mon Sep 17 00:00:00 2001 From: Kaushik Bokka Date: Sun, 7 Mar 2021 03:06:39 +0530 Subject: [PATCH 03/10] update torchvision version --- requirements/models.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/models.txt b/requirements/models.txt index a92a7ef6bd..af91bce1a2 100644 --- a/requirements/models.txt +++ b/requirements/models.txt @@ -1,4 +1,4 @@ -torchvision>=0.7 +torchvision==0.9.0 scikit-learn>=0.23 Pillow opencv-python From 4380cec288cf8a9eec4ec5d226e5e1d6e4b80407 Mon Sep 17 00:00:00 2001 From: Kaushik Bokka Date: Sun, 7 Mar 2021 03:39:00 +0530 Subject: [PATCH 04/10] revert back torch versions --- requirements.txt | 2 +- requirements/models.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d5e1b90c11..4d3da28520 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -torch==1.8 +torch>=1.6 pytorch-lightning>=1.1.1, <1.2 \ No newline at end of file diff --git a/requirements/models.txt b/requirements/models.txt index af91bce1a2..a92a7ef6bd 100644 --- a/requirements/models.txt +++ b/requirements/models.txt @@ -1,4 +1,4 @@ -torchvision==0.9.0 +torchvision>=0.7 scikit-learn>=0.23 Pillow opencv-python From 823c0414266e342ccc76d900d34c0d94060b95b4 Mon Sep 17 00:00:00 2001 From: Kaushik Bokka Date: Sun, 7 Mar 2021 14:41:50 +0530 Subject: [PATCH 05/10] add user agent --- pl_bolts/models/mnist_module.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pl_bolts/models/mnist_module.py b/pl_bolts/models/mnist_module.py index be6c4ee623..ef0e274d0d 100644 --- a/pl_bolts/models/mnist_module.py +++ b/pl_bolts/models/mnist_module.py @@ -1,4 +1,5 @@ from argparse import ArgumentParser +from six.moves import urllib import torch from pytorch_lightning import LightningModule, Trainer @@ -14,6 +15,11 @@ else: # pragma: no cover warn_missing_pkg('torchvision') +# TorchVision hotfix https://github.com/pytorch/vision/issues/1938 +opener = urllib.request.build_opener() +opener.addheaders = [('User-agent', 'Mozilla/5.0')] +urllib.request.install_opener(opener) + class LitMNIST(LightningModule): From 15e64d8c90dff9650dafc9ddcaa06dd80d9973b8 Mon Sep 17 00:00:00 2001 From: Kaushik Bokka Date: Sun, 7 Mar 2021 20:08:41 +0530 Subject: [PATCH 06/10] fix expected od loss tensor shape --- tests/losses/test_object_detection.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/losses/test_object_detection.py b/tests/losses/test_object_detection.py index ac4ac3083d..26f1231cdf 100644 --- a/tests/losses/test_object_detection.py +++ b/tests/losses/test_object_detection.py @@ -10,7 +10,7 @@ @pytest.mark.parametrize( "preds, target, expected_loss", - [(torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 100, 200, 200]]), torch.tensor([0.0]))] + [(torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 100, 200, 200]]), torch.tensor([[0.0]]))] ) def test_iou_complete_overlap(preds, target, expected_loss): torch.testing.assert_allclose(iou_loss(preds, target), expected_loss) @@ -18,8 +18,8 @@ def test_iou_complete_overlap(preds, target, expected_loss): @pytest.mark.parametrize( "preds, target, expected_loss", [ - (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 200, 200, 300]]), torch.tensor([1.0])), - (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[200, 200, 300, 300]]), torch.tensor([1.0])), + (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 200, 200, 300]]), torch.tensor([[1.0]])), + (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[200, 200, 300, 300]]), torch.tensor([[1.0]])), ] ) def test_iou_no_overlap(preds, target, expected_loss): @@ -28,7 +28,7 @@ def test_iou_no_overlap(preds, target, expected_loss): @pytest.mark.parametrize( "preds, target, expected_loss", - [(torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 100, 200, 200]]), torch.tensor([0.0]))] + [(torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 100, 200, 200]]), torch.tensor([[0.0]]))] ) def test_complete_overlap(preds, target, expected_loss): torch.testing.assert_allclose(giou_loss(preds, target), expected_loss) @@ -36,8 +36,8 @@ def test_complete_overlap(preds, target, expected_loss): @pytest.mark.parametrize( "preds, target, expected_loss", [ - (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 200, 200, 300]]), torch.tensor([1.0])), - (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[200, 200, 300, 300]]), torch.tensor([1.5])), + (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 200, 200, 300]]), torch.tensor([[1.0]])), + (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[200, 200, 300, 300]]), torch.tensor([[1.5]])), ] ) def test_no_overlap(preds, target, expected_loss): From 3c2a2a5bf0b05b539ba98a86d2e6420dcf8cee7e Mon Sep 17 00:00:00 2001 From: Kaushik Bokka Date: Sun, 7 Mar 2021 20:24:57 +0530 Subject: [PATCH 07/10] fix expected od loss tensor shape --- tests/metrics/test_object_detection.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/metrics/test_object_detection.py b/tests/metrics/test_object_detection.py index 83765012ca..606cbcc1b6 100644 --- a/tests/metrics/test_object_detection.py +++ b/tests/metrics/test_object_detection.py @@ -10,7 +10,7 @@ @pytest.mark.parametrize( "preds, target, expected_iou", - [(torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 100, 200, 200]]), torch.tensor([1.0]))] + [(torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 100, 200, 200]]), torch.tensor([[1.0]]))] ) def test_iou_complete_overlap(preds, target, expected_iou): torch.testing.assert_allclose(iou(preds, target), expected_iou) @@ -18,8 +18,8 @@ def test_iou_complete_overlap(preds, target, expected_iou): @pytest.mark.parametrize( "preds, target, expected_iou", [ - (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 200, 200, 300]]), torch.tensor([0.0])), - (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[200, 200, 300, 300]]), torch.tensor([0.0])), + (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 200, 200, 300]]), torch.tensor([[0.0]])), + (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[200, 200, 300, 300]]), torch.tensor([[0.0]])), ] ) def test_iou_no_overlap(preds, target, expected_iou): @@ -39,7 +39,7 @@ def test_iou_multi(preds, target, expected_iou): @pytest.mark.parametrize( "preds, target, expected_giou", - [(torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 100, 200, 200]]), torch.tensor([1.0]))] + [(torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 100, 200, 200]]), torch.tensor([[1.0]]))] ) def test_complete_overlap(preds, target, expected_giou): torch.testing.assert_allclose(giou(preds, target), expected_giou) @@ -47,8 +47,8 @@ def test_complete_overlap(preds, target, expected_giou): @pytest.mark.parametrize( "preds, target, expected_giou", [ - (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 200, 200, 300]]), torch.tensor([0.0])), - (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[200, 200, 300, 300]]), torch.tensor([-0.5])), + (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[100, 200, 200, 300]]), torch.tensor([[0.0]])), + (torch.tensor([[100, 100, 200, 200]]), torch.tensor([[200, 200, 300, 300]]), torch.tensor([[-0.5]])), ] ) def test_no_overlap(preds, target, expected_giou): From f08c38535a1289eb4579969c19f7ccf99adc7bda Mon Sep 17 00:00:00 2001 From: Kaushik Bokka Date: Sun, 7 Mar 2021 20:41:22 +0530 Subject: [PATCH 08/10] fix unused imports --- pl_bolts/datasets/imagenet_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_bolts/datasets/imagenet_dataset.py b/pl_bolts/datasets/imagenet_dataset.py index 607af8cbf7..ab06ac0fce 100644 --- a/pl_bolts/datasets/imagenet_dataset.py +++ b/pl_bolts/datasets/imagenet_dataset.py @@ -2,7 +2,7 @@ import hashlib import os import shutil -import sys +import sys # noqa F401 import tarfile import tempfile import zipfile From 2504997bcc5c7b838f5e51f51a893f68f4686c5c Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 9 Mar 2021 11:04:19 +0100 Subject: [PATCH 09/10] move --- pl_bolts/datasets/__init__.py | 8 ++++++++ pl_bolts/models/mnist_module.py | 6 ------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pl_bolts/datasets/__init__.py b/pl_bolts/datasets/__init__.py index 081e78f149..6da023dfae 100644 --- a/pl_bolts/datasets/__init__.py +++ b/pl_bolts/datasets/__init__.py @@ -1,3 +1,5 @@ +import urllib + from pl_bolts.datasets.base_dataset import LightDataset from pl_bolts.datasets.cifar10_dataset import CIFAR10, TrialCIFAR10 from pl_bolts.datasets.concat_dataset import ConcatDataset @@ -31,3 +33,9 @@ "CIFAR10Mixed", "SSLDatasetMixin", ] + +# TorchVision hotfix https://github.com/pytorch/vision/issues/1938 +opener = urllib.request.build_opener() +opener.addheaders = [('User-agent', 'Mozilla/5.0')] +urllib.request.install_opener(opener) + diff --git a/pl_bolts/models/mnist_module.py b/pl_bolts/models/mnist_module.py index ef0e274d0d..be6c4ee623 100644 --- a/pl_bolts/models/mnist_module.py +++ b/pl_bolts/models/mnist_module.py @@ -1,5 +1,4 @@ from argparse import ArgumentParser -from six.moves import urllib import torch from pytorch_lightning import LightningModule, Trainer @@ -15,11 +14,6 @@ else: # pragma: no cover warn_missing_pkg('torchvision') -# TorchVision hotfix https://github.com/pytorch/vision/issues/1938 -opener = urllib.request.build_opener() -opener.addheaders = [('User-agent', 'Mozilla/5.0')] -urllib.request.install_opener(opener) - class LitMNIST(LightningModule): From efe5de2cc3d25bd6d103e28368e328cb227439b7 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 9 Mar 2021 11:08:39 +0100 Subject: [PATCH 10/10] format --- pl_bolts/datasets/__init__.py | 1 - tests/models/test_classic_ml.py | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pl_bolts/datasets/__init__.py b/pl_bolts/datasets/__init__.py index 6da023dfae..191f37c10a 100644 --- a/pl_bolts/datasets/__init__.py +++ b/pl_bolts/datasets/__init__.py @@ -38,4 +38,3 @@ opener = urllib.request.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] urllib.request.install_opener(opener) - diff --git a/tests/models/test_classic_ml.py b/tests/models/test_classic_ml.py index 197c3170a0..513c589f46 100644 --- a/tests/models/test_classic_ml.py +++ b/tests/models/test_classic_ml.py @@ -20,8 +20,13 @@ def test_linear_regression_model(tmpdir): loader = DataLoader(SklearnDataset(X, y), batch_size=2) model = LinearRegression(input_dim=2, learning_rate=0.5) - trainer = pl.Trainer(max_epochs=300, default_root_dir=tmpdir, - progress_bar_refresh_rate=0, logger=False, checkpoint_callback=False) + trainer = pl.Trainer( + max_epochs=300, + default_root_dir=tmpdir, + progress_bar_refresh_rate=0, + logger=False, + checkpoint_callback=False, + ) trainer.fit( model, loader,