From 8676eeea68f064a70447fe3e875e2fec345805aa Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 4 Jul 2019 12:16:28 +0200 Subject: [PATCH 1/5] cast images to PIL at instantiation instead of runtime --- torchvision/datasets/cifar.py | 8 ++++---- torchvision/datasets/svhn.py | 24 ++++++++++-------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/torchvision/datasets/cifar.py b/torchvision/datasets/cifar.py index f1279fa9ae7..b5d515cea78 100644 --- a/torchvision/datasets/cifar.py +++ b/torchvision/datasets/cifar.py @@ -94,6 +94,10 @@ def __init__(self, root, train=True, self.data = np.vstack(self.data).reshape(-1, 3, 32, 32) self.data = self.data.transpose((0, 2, 3, 1)) # convert to HWC + # doing this so that it is consistent with all other datasets + # to return a PIL Image + self.data = [Image.fromarray(image) for image in self.data] + self._load_meta() def _load_meta(self): @@ -119,10 +123,6 @@ def __getitem__(self, index): """ img, target = self.data[index], self.targets[index] - # doing this so that it is consistent with all other datasets - # to return a PIL Image - img = Image.fromarray(img) - if self.transform is not None: img = self.transform(img) diff --git a/torchvision/datasets/svhn.py b/torchvision/datasets/svhn.py index 972125d74fb..bda48d6afb8 100644 --- a/torchvision/datasets/svhn.py +++ b/torchvision/datasets/svhn.py @@ -66,20 +66,20 @@ def __init__(self, root, split='train', import scipy.io as sio # reading(loading) mat file as array - loaded_mat = sio.loadmat(os.path.join(self.root, self.filename)) + loaded_mat = sio.loadmat(os.path.join(self.root, self.filename), + squeeze_me=True) + data, targets = loaded_mat['X'], loaded_mat['y'] - self.data = loaded_mat['X'] - # loading from the .mat file gives an np array of type np.uint8 - # converting to np.int64, so that we have a LongTensor after - # the conversion from the numpy array - # the squeeze is needed to obtain a 1D tensor - self.labels = loaded_mat['y'].astype(np.int64).squeeze() + # doing this so that it is consistent with all other datasets + # to return a PIL Image + self.data = [Image.fromarray(image.squeeze(3)) + for image in np.split(data, len(targets), axis=3)] # the svhn dataset assigns the class label "10" to the digit 0 # this makes it inconsistent with several loss functions # which expect the class labels to be in the range [0, C-1] - np.place(self.labels, self.labels == 10, 0) - self.data = np.transpose(self.data, (3, 2, 0, 1)) + np.place(targets, targets == 10, 0) + self.targets = [int(target) for target in targets] def __getitem__(self, index): """ @@ -89,11 +89,7 @@ def __getitem__(self, index): Returns: tuple: (image, target) where target is index of the target class. """ - img, target = self.data[index], int(self.labels[index]) - - # doing this so that it is consistent with all other datasets - # to return a PIL Image - img = Image.fromarray(np.transpose(img, (1, 2, 0))) + img, target = self.data[index], self.targets[index] if self.transform is not None: img = self.transform(img) From c8bb36d7db75b60f23b4fd9da6847770eaf31a94 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 4 Jul 2019 12:16:45 +0200 Subject: [PATCH 2/5] add test for svhn --- test/fakedata_generation.py | 16 ++++++++++++++++ test/test_datasets.py | 10 +++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/test/fakedata_generation.py b/test/fakedata_generation.py index bef63a37b05..afd0bc975ed 100644 --- a/test/fakedata_generation.py +++ b/test/fakedata_generation.py @@ -241,3 +241,19 @@ def _make_polygon_target(file): '{city}_000000_000000_leftImg8bit.png'.format(city=city))) yield tmp_dir + +@contextlib.contextmanager +def svhn_root(): + import scipy.io as sio + + def _make_mat(file): + images = np.zeros((32, 32, 3, 2), dtype=np.uint8) + targets = np.zeros((2,), dtype=np.uint8) + sio.savemat(file, {'X': images, 'y': targets}) + + with get_tmp_dir() as root: + _make_mat(os.path.join(root, "train_32x32.mat")) + _make_mat(os.path.join(root, "test_32x32.mat")) + _make_mat(os.path.join(root, "extra_32x32.mat")) + + yield root diff --git a/test/test_datasets.py b/test/test_datasets.py index 19914b5e2d1..f9b46e72ea1 100644 --- a/test/test_datasets.py +++ b/test/test_datasets.py @@ -7,7 +7,8 @@ from torch._utils_internal import get_file_path_2 import torchvision from common_utils import get_tmp_dir -from fakedata_generation import mnist_root, cifar_root, imagenet_root, cityscapes_root +from fakedata_generation import mnist_root, cifar_root, imagenet_root, \ + cityscapes_root, svhn_root class Tester(unittest.TestCase): @@ -185,6 +186,13 @@ def test_cityscapes(self): self.assertTrue(isinstance(output[1][1], dict)) # polygon self.assertTrue(isinstance(output[1][2], PIL.Image.Image)) # color + @mock.patch('torchvision.datasets.SVHN._check_integrity') + def test_svhn(self, mock_check): + mock_check.return_value = True + with svhn_root() as root: + dataset = torchvision.datasets.SVHN(root, split="train") + self.generic_classification_dataset_test(dataset, num_images=2) + if __name__ == '__main__': unittest.main() From 98608678ec64856287bade258d06ee936653ca04 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 4 Jul 2019 12:19:08 +0200 Subject: [PATCH 3/5] added tests for remaining SVHN splits --- test/test_datasets.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_datasets.py b/test/test_datasets.py index f9b46e72ea1..0c96df11e4e 100644 --- a/test/test_datasets.py +++ b/test/test_datasets.py @@ -193,6 +193,12 @@ def test_svhn(self, mock_check): dataset = torchvision.datasets.SVHN(root, split="train") self.generic_classification_dataset_test(dataset, num_images=2) + dataset = torchvision.datasets.SVHN(root, split="test") + self.generic_classification_dataset_test(dataset, num_images=2) + + dataset = torchvision.datasets.SVHN(root, split="extra") + self.generic_classification_dataset_test(dataset, num_images=2) + if __name__ == '__main__': unittest.main() From d28ab269cd4aefc3064d1441d707bcac93fbad1e Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 4 Jul 2019 14:07:30 +0200 Subject: [PATCH 4/5] flake8 --- test/fakedata_generation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/fakedata_generation.py b/test/fakedata_generation.py index afd0bc975ed..d14bc0c8304 100644 --- a/test/fakedata_generation.py +++ b/test/fakedata_generation.py @@ -242,6 +242,7 @@ def _make_polygon_target(file): yield tmp_dir + @contextlib.contextmanager def svhn_root(): import scipy.io as sio From a90324aa5e8f703dfa5b366a39fce59ea05ba73c Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Tue, 9 Jul 2019 11:41:50 +0200 Subject: [PATCH 5/5] rolled back changes to datasets --- torchvision/datasets/cifar.py | 8 ++++---- torchvision/datasets/svhn.py | 24 ++++++++++++++---------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/torchvision/datasets/cifar.py b/torchvision/datasets/cifar.py index b5d515cea78..f1279fa9ae7 100644 --- a/torchvision/datasets/cifar.py +++ b/torchvision/datasets/cifar.py @@ -94,10 +94,6 @@ def __init__(self, root, train=True, self.data = np.vstack(self.data).reshape(-1, 3, 32, 32) self.data = self.data.transpose((0, 2, 3, 1)) # convert to HWC - # doing this so that it is consistent with all other datasets - # to return a PIL Image - self.data = [Image.fromarray(image) for image in self.data] - self._load_meta() def _load_meta(self): @@ -123,6 +119,10 @@ def __getitem__(self, index): """ img, target = self.data[index], self.targets[index] + # doing this so that it is consistent with all other datasets + # to return a PIL Image + img = Image.fromarray(img) + if self.transform is not None: img = self.transform(img) diff --git a/torchvision/datasets/svhn.py b/torchvision/datasets/svhn.py index bda48d6afb8..972125d74fb 100644 --- a/torchvision/datasets/svhn.py +++ b/torchvision/datasets/svhn.py @@ -66,20 +66,20 @@ def __init__(self, root, split='train', import scipy.io as sio # reading(loading) mat file as array - loaded_mat = sio.loadmat(os.path.join(self.root, self.filename), - squeeze_me=True) - data, targets = loaded_mat['X'], loaded_mat['y'] + loaded_mat = sio.loadmat(os.path.join(self.root, self.filename)) - # doing this so that it is consistent with all other datasets - # to return a PIL Image - self.data = [Image.fromarray(image.squeeze(3)) - for image in np.split(data, len(targets), axis=3)] + self.data = loaded_mat['X'] + # loading from the .mat file gives an np array of type np.uint8 + # converting to np.int64, so that we have a LongTensor after + # the conversion from the numpy array + # the squeeze is needed to obtain a 1D tensor + self.labels = loaded_mat['y'].astype(np.int64).squeeze() # the svhn dataset assigns the class label "10" to the digit 0 # this makes it inconsistent with several loss functions # which expect the class labels to be in the range [0, C-1] - np.place(targets, targets == 10, 0) - self.targets = [int(target) for target in targets] + np.place(self.labels, self.labels == 10, 0) + self.data = np.transpose(self.data, (3, 2, 0, 1)) def __getitem__(self, index): """ @@ -89,7 +89,11 @@ def __getitem__(self, index): Returns: tuple: (image, target) where target is index of the target class. """ - img, target = self.data[index], self.targets[index] + img, target = self.data[index], int(self.labels[index]) + + # doing this so that it is consistent with all other datasets + # to return a PIL Image + img = Image.fromarray(np.transpose(img, (1, 2, 0))) if self.transform is not None: img = self.transform(img)