From 72ccc5cbd81e66af344ce2bfb1821c3a9dcd7ebd Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Fri, 18 Feb 2022 23:09:51 +0530 Subject: [PATCH 01/53] basic stanford cars structure added. --- .../_builtin/stanford_cars.categories | 196 ++++++++++++++++++ .../datasets/_builtin/stanford_cars.py | 0 2 files changed, 196 insertions(+) create mode 100644 torchvision/prototype/datasets/_builtin/stanford_cars.categories create mode 100644 torchvision/prototype/datasets/_builtin/stanford_cars.py diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.categories b/torchvision/prototype/datasets/_builtin/stanford_cars.categories new file mode 100644 index 00000000000..54aa513e43c --- /dev/null +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.categories @@ -0,0 +1,196 @@ +AM General Hummer SUV 2000 +Acura RL Sedan 2012 +Acura TL Sedan 2012 +Acura TL Type-S 2008 +Acura TSX Sedan 2012 +Acura Integra Type R 2001 +Acura ZDX Hatchback 2012 +Aston Martin V8 Vantage Convertible 2012 +Aston Martin V8 Vantage Coupe 2012 +Aston Martin Virage Convertible 2012 +Aston Martin Virage Coupe 2012 +Audi RS 4 Convertible 2008 +Audi A5 Coupe 2012 +Audi TTS Coupe 2012 +Audi R8 Coupe 2012 +Audi V8 Sedan 1994 +Audi 100 Sedan 1994 +Audi 100 Wagon 1994 +Audi TT Hatchback 2011 +Audi S6 Sedan 2011 +Audi S5 Convertible 2012 +Audi S5 Coupe 2012 +Audi S4 Sedan 2012 +Audi S4 Sedan 2007 +Audi TT RS Coupe 2012 +BMW ActiveHybrid 5 Sedan 2012 +BMW 1 Series Convertible 2012 +BMW 1 Series Coupe 2012 +BMW 3 Series Sedan 2012 +BMW 3 Series Wagon 2012 +BMW 6 Series Convertible 2007 +BMW X5 SUV 2007 +BMW X6 SUV 2012 +BMW M3 Coupe 2012 +BMW M5 Sedan 2010 +BMW M6 Convertible 2010 +BMW X3 SUV 2012 +BMW Z4 Convertible 2012 +Bentley Continental Supersports Conv. Convertible 2012 +Bentley Arnage Sedan 2009 +Bentley Mulsanne Sedan 2011 +Bentley Continental GT Coupe 2012 +Bentley Continental GT Coupe 2007 +Bentley Continental Flying Spur Sedan 2007 +Bugatti Veyron 16.4 Convertible 2009 +Bugatti Veyron 16.4 Coupe 2009 +Buick Regal GS 2012 +Buick Rainier SUV 2007 +Buick Verano Sedan 2012 +Buick Enclave SUV 2012 +Cadillac CTS-V Sedan 2012 +Cadillac SRX SUV 2012 +Cadillac Escalade EXT Crew Cab 2007 +Chevrolet Silverado 1500 Hybrid Crew Cab 2012 +Chevrolet Corvette Convertible 2012 +Chevrolet Corvette ZR1 2012 +Chevrolet Corvette Ron Fellows Edition Z06 2007 +Chevrolet Traverse SUV 2012 +Chevrolet Camaro Convertible 2012 +Chevrolet HHR SS 2010 +Chevrolet Impala Sedan 2007 +Chevrolet Tahoe Hybrid SUV 2012 +Chevrolet Sonic Sedan 2012 +Chevrolet Express Cargo Van 2007 +Chevrolet Avalanche Crew Cab 2012 +Chevrolet Cobalt SS 2010 +Chevrolet Malibu Hybrid Sedan 2010 +Chevrolet TrailBlazer SS 2009 +Chevrolet Silverado 2500HD Regular Cab 2012 +Chevrolet Silverado 1500 Classic Extended Cab 2007 +Chevrolet Express Van 2007 +Chevrolet Monte Carlo Coupe 2007 +Chevrolet Malibu Sedan 2007 +Chevrolet Silverado 1500 Extended Cab 2012 +Chevrolet Silverado 1500 Regular Cab 2012 +Chrysler Aspen SUV 2009 +Chrysler Sebring Convertible 2010 +Chrysler Town and Country Minivan 2012 +Chrysler 300 SRT-8 2010 +Chrysler Crossfire Convertible 2008 +Chrysler PT Cruiser Convertible 2008 +Daewoo Nubira Wagon 2002 +Dodge Caliber Wagon 2012 +Dodge Caliber Wagon 2007 +Dodge Caravan Minivan 1997 +Dodge Ram Pickup 3500 Crew Cab 2010 +Dodge Ram Pickup 3500 Quad Cab 2009 +Dodge Sprinter Cargo Van 2009 +Dodge Journey SUV 2012 +Dodge Dakota Crew Cab 2010 +Dodge Dakota Club Cab 2007 +Dodge Magnum Wagon 2008 +Dodge Challenger SRT8 2011 +Dodge Durango SUV 2012 +Dodge Durango SUV 2007 +Dodge Charger Sedan 2012 +Dodge Charger SRT-8 2009 +Eagle Talon Hatchback 1998 +FIAT 500 Abarth 2012 +FIAT 500 Convertible 2012 +Ferrari FF Coupe 2012 +Ferrari California Convertible 2012 +Ferrari 458 Italia Convertible 2012 +Ferrari 458 Italia Coupe 2012 +Fisker Karma Sedan 2012 +Ford F-450 Super Duty Crew Cab 2012 +Ford Mustang Convertible 2007 +Ford Freestar Minivan 2007 +Ford Expedition EL SUV 2009 +Ford Edge SUV 2012 +Ford Ranger SuperCab 2011 +Ford GT Coupe 2006 +Ford F-150 Regular Cab 2012 +Ford F-150 Regular Cab 2007 +Ford Focus Sedan 2007 +Ford E-Series Wagon Van 2012 +Ford Fiesta Sedan 2012 +GMC Terrain SUV 2012 +GMC Savana Van 2012 +GMC Yukon Hybrid SUV 2012 +GMC Acadia SUV 2012 +GMC Canyon Extended Cab 2012 +Geo Metro Convertible 1993 +HUMMER H3T Crew Cab 2010 +HUMMER H2 SUT Crew Cab 2009 +Honda Odyssey Minivan 2012 +Honda Odyssey Minivan 2007 +Honda Accord Coupe 2012 +Honda Accord Sedan 2012 +Hyundai Veloster Hatchback 2012 +Hyundai Santa Fe SUV 2012 +Hyundai Tucson SUV 2012 +Hyundai Veracruz SUV 2012 +Hyundai Sonata Hybrid Sedan 2012 +Hyundai Elantra Sedan 2007 +Hyundai Accent Sedan 2012 +Hyundai Genesis Sedan 2012 +Hyundai Sonata Sedan 2012 +Hyundai Elantra Touring Hatchback 2012 +Hyundai Azera Sedan 2012 +Infiniti G Coupe IPL 2012 +Infiniti QX56 SUV 2011 +Isuzu Ascender SUV 2008 +Jaguar XK XKR 2012 +Jeep Patriot SUV 2012 +Jeep Wrangler SUV 2012 +Jeep Liberty SUV 2012 +Jeep Grand Cherokee SUV 2012 +Jeep Compass SUV 2012 +Lamborghini Reventon Coupe 2008 +Lamborghini Aventador Coupe 2012 +Lamborghini Gallardo LP 570-4 Superleggera 2012 +Lamborghini Diablo Coupe 2001 +Land Rover Range Rover SUV 2012 +Land Rover LR2 SUV 2012 +Lincoln Town Car Sedan 2011 +MINI Cooper Roadster Convertible 2012 +Maybach Landaulet Convertible 2012 +Mazda Tribute SUV 2011 +McLaren MP4-12C Coupe 2012 +Mercedes-Benz 300-Class Convertible 1993 +Mercedes-Benz C-Class Sedan 2012 +Mercedes-Benz SL-Class Coupe 2009 +Mercedes-Benz E-Class Sedan 2012 +Mercedes-Benz S-Class Sedan 2012 +Mercedes-Benz Sprinter Van 2012 +Mitsubishi Lancer Sedan 2012 +Nissan Leaf Hatchback 2012 +Nissan NV Passenger Van 2012 +Nissan Juke Hatchback 2012 +Nissan 240SX Coupe 1998 +Plymouth Neon Coupe 1999 +Porsche Panamera Sedan 2012 +Ram C/V Cargo Van Minivan 2012 +Rolls-Royce Phantom Drophead Coupe Convertible 2012 +Rolls-Royce Ghost Sedan 2012 +Rolls-Royce Phantom Sedan 2012 +Scion xD Hatchback 2012 +Spyker C8 Convertible 2009 +Spyker C8 Coupe 2009 +Suzuki Aerio Sedan 2007 +Suzuki Kizashi Sedan 2012 +Suzuki SX4 Hatchback 2012 +Suzuki SX4 Sedan 2012 +Tesla Model S Sedan 2012 +Toyota Sequoia SUV 2012 +Toyota Camry Sedan 2012 +Toyota Corolla Sedan 2012 +Toyota 4Runner SUV 2012 +Volkswagen Golf Hatchback 2012 +Volkswagen Golf Hatchback 1991 +Volkswagen Beetle Hatchback 2012 +Volvo C30 Hatchback 2012 +Volvo 240 Sedan 1993 +Volvo XC90 SUV 2007 +smart fortwo Convertible 2012 \ No newline at end of file diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py new file mode 100644 index 00000000000..e69de29bb2d From 1ce692cbe5b506fcbd0310b152feecb420393d69 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Fri, 18 Feb 2022 23:10:25 +0530 Subject: [PATCH 02/53] added class in the init. --- torchvision/prototype/datasets/_builtin/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torchvision/prototype/datasets/_builtin/__init__.py b/torchvision/prototype/datasets/_builtin/__init__.py index 9fdfca904f5..c84804ae536 100644 --- a/torchvision/prototype/datasets/_builtin/__init__.py +++ b/torchvision/prototype/datasets/_builtin/__init__.py @@ -1,3 +1,4 @@ +from torch import imag from .caltech import Caltech101, Caltech256 from .celeba import CelebA from .cifar import Cifar10, Cifar100 @@ -15,3 +16,4 @@ from .semeion import SEMEION from .svhn import SVHN from .voc import VOC +from .stanford_cars import StanfordCars From 20794e2e9ac643912135b96a7601a4119054d3d1 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Fri, 18 Feb 2022 23:24:28 +0530 Subject: [PATCH 03/53] how to decode the images --- .../datasets/_builtin/stanford_cars.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index e69de29bb2d..a7e00042a6b 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -0,0 +1,111 @@ + +from cProfile import label +from hashlib import sha256 +import io +from collections import namedtuple +from pickletools import read_string1 +from typing import Any, Callable, Dict, List, Optional, Tuple, Iterator +import pathlib +from torchdata.datapipes.iter import ( + IterDataPipe, + Mapper, + Filter, + ) + +from torchvision.prototype.datasets.utils._internal import read_mat,hint_sharding, hint_shuffling, path_comparator +import torch +from torchdata.datapipes.iter import IterDataPipe, Mapper, Zipper +from torchvision.prototype import features +from torchvision.prototype.features import Label, BoundingBox, EncodedImage +from torchvision.prototype.datasets.utils import ( + Dataset, + DatasetConfig, + DatasetInfo, + OnlineResource, + DatasetType, + HttpResource +) + + +from torchvision.prototype.datasets.utils._internal import ( + hint_sharding, + hint_shuffling, +) +from torchvision.prototype.features import Label + + +class StanfordCars(Dataset): + def _make_info(self) -> DatasetInfo: + return DatasetInfo( + name="stanford_cars", + type=DatasetType.IMAGE, + homepage="https://ai.stanford.edu/~jkrause/cars/car_dataset.html", + dependencies=("scipy",), + valid_options=dict(split=("test","train"), + ), + ) + + _URL_ROOT = "https://ai.stanford.edu/~jkrause/" + _URLS = { + "train":f"{_URL_ROOT}car196/cars_train.tgz", + "test":f"{_URL_ROOT}car196/cars_test.tgz", + "test_ground_truth":f"{_URL_ROOT}car196/cars_test_annos_withlabels.mat", + "devkit":f"{_URL_ROOT}cars/car_devkit.tgz" + } + + _CHECKSUM = { + "train": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", + "test": "bffea656d6f425cba3c91c6d83336e4c5f86c6cffd8975b0f375d3a10da8e243", + "test_ground_truth": "790f75be8ea34eeded134cc559332baf23e30e91367e9ddca97d26ed9b895f05", + "devkit": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", + } + + def resources(self, config: DatasetConfig) -> List[OnlineResource]: + resources = [HttpResource(self._URLS[config.split] , sha256=self._CHECKSUM[config.split])] + print(len(resources)) + if config.split == "test": + resources.append(HttpResource(self._URLS["test_ground_truth"],sha256=self._CHECKSUM["test_ground_truth"])) + else: + resources.append(HttpResource(url=self._URLS["devkit"],sha256=self._CHECKSUM["devkit"])) + return resources + + def _make_datapipe( + self, + resource_dps: List[IterDataPipe], + *, + config: DatasetConfig, + decoder: Optional[Callable[[io.IOBase], torch.Tensor]], + ) -> IterDataPipe[Dict[str, Any]] : + + images_dp, targets_dp = resource_dps + + if config.split == "train": + targets_dp = Filter(targets_dp, path_comparator("name", "cars_train_annos.mat")) + + + #right now our both dps are of .tgz and .mat format; for all configs(train and test.) + targets_dp = Mapper(targets_dp,self._read_labels) + dp =Zipper(images_dp,targets_dp) + dp=hint_sharding(dp) + dp=hint_shuffling(dp) + + + return Mapper(dp,self._collate_and_decode) + + def _collate_and_decode(self, data: Tuple[Any, Any]) -> Dict[str, Any]: + image, target = data # They're both numpy arrays at this point + + image_path, image_buffer = image + + image = EncodedImage.from_file(image_buffer) + + label = Label(target.item()) + return {"image": image, "label":label} + + def _read_labels(self,data): + labels = data[0] + labels = read_mat(labels,squeeze_me=True) + labels=labels["annotations"] + bboxs_and_class = zip(labels["bbox_x1"],labels["bbox_y1"],labels["bbox_x2"],labels["bbox_y2"],labels["class"]) + + return bboxs_and_class From e4c3f9d9886e8c012a756c53c2644e1d389d7ac4 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Fri, 18 Feb 2022 23:38:19 +0530 Subject: [PATCH 04/53] fix typo in init --- .../prototype/datasets/_builtin/__init__.py | 1 - .../datasets/_builtin/stanford_cars.py | 13 +++++----- trial.py | 25 +++++++++++++++++++ 3 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 trial.py diff --git a/torchvision/prototype/datasets/_builtin/__init__.py b/torchvision/prototype/datasets/_builtin/__init__.py index c84804ae536..a6a647dd042 100644 --- a/torchvision/prototype/datasets/_builtin/__init__.py +++ b/torchvision/prototype/datasets/_builtin/__init__.py @@ -1,4 +1,3 @@ -from torch import imag from .caltech import Caltech101, Caltech256 from .celeba import CelebA from .cifar import Cifar10, Cifar100 diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index a7e00042a6b..844dfcaa566 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -16,13 +16,13 @@ import torch from torchdata.datapipes.iter import IterDataPipe, Mapper, Zipper from torchvision.prototype import features -from torchvision.prototype.features import Label, BoundingBox, EncodedImage +from torchvision.prototype.features import Label, BoundingBox, _Feature, EncodedImage from torchvision.prototype.datasets.utils import ( Dataset, DatasetConfig, DatasetInfo, OnlineResource, - DatasetType, + #DatasetType, HttpResource ) @@ -38,7 +38,7 @@ class StanfordCars(Dataset): def _make_info(self) -> DatasetInfo: return DatasetInfo( name="stanford_cars", - type=DatasetType.IMAGE, + #type=DatasetType.IMAGE, homepage="https://ai.stanford.edu/~jkrause/cars/car_dataset.html", dependencies=("scipy",), valid_options=dict(split=("test","train"), @@ -74,7 +74,6 @@ def _make_datapipe( resource_dps: List[IterDataPipe], *, config: DatasetConfig, - decoder: Optional[Callable[[io.IOBase], torch.Tensor]], ) -> IterDataPipe[Dict[str, Any]] : images_dp, targets_dp = resource_dps @@ -98,14 +97,14 @@ def _collate_and_decode(self, data: Tuple[Any, Any]) -> Dict[str, Any]: image_path, image_buffer = image image = EncodedImage.from_file(image_buffer) - - label = Label(target.item()) - return {"image": image, "label":label} + def _read_labels(self,data): labels = data[0] labels = read_mat(labels,squeeze_me=True) labels=labels["annotations"] + print("\n"*10) + print(len(labels["class"])) bboxs_and_class = zip(labels["bbox_x1"],labels["bbox_y1"],labels["bbox_x2"],labels["bbox_y2"],labels["class"]) return bboxs_and_class diff --git a/trial.py b/trial.py new file mode 100644 index 00000000000..bc0deecf237 --- /dev/null +++ b/trial.py @@ -0,0 +1,25 @@ +#from lib2to3.pgen2.token import LSQB +#from tkinter import Label +from distutils.command.config import config +from torchvision.prototype import datasets +#from torchvision.prototype.datasets._builtin.stanford_cars import StanfordCars +dataset= datasets.load("stanford_cars") +ninja2 =0 +#print(type(dataset)) +print(dataset, "jojo") +#print(label) +#print(dat +#print(len(dataset[0])) +for sample in dataset: + #print(len(sample) + print(sample,"is a saple00") + #print(ninja,"hehe") + #print(c,d,e) + ninja2+=1 + if ninja2>10: + break +# print(sample) +# print(label)# this is the content of an item in datapipe returned by _make_datapipe() +# if ninja>29: +# break +# ninja+=2 \ No newline at end of file From 9ae5f233d225d7d8fea1343f5f145bde6f7e7849 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 19 Feb 2022 00:39:33 +0530 Subject: [PATCH 05/53] works partially --- .../datasets/_builtin/stanford_cars.py | 44 ++++++++++--------- trial.py | 25 ----------- 2 files changed, 23 insertions(+), 46 deletions(-) delete mode 100644 trial.py diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 844dfcaa566..46a283b3032 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -6,12 +6,13 @@ from pickletools import read_string1 from typing import Any, Callable, Dict, List, Optional, Tuple, Iterator import pathlib +from unicodedata import category from torchdata.datapipes.iter import ( IterDataPipe, Mapper, Filter, ) - +import numpy as np from torchvision.prototype.datasets.utils._internal import read_mat,hint_sharding, hint_shuffling, path_comparator import torch from torchdata.datapipes.iter import IterDataPipe, Mapper, Zipper @@ -62,7 +63,6 @@ def _make_info(self) -> DatasetInfo: def resources(self, config: DatasetConfig) -> List[OnlineResource]: resources = [HttpResource(self._URLS[config.split] , sha256=self._CHECKSUM[config.split])] - print(len(resources)) if config.split == "test": resources.append(HttpResource(self._URLS["test_ground_truth"],sha256=self._CHECKSUM["test_ground_truth"])) else: @@ -77,34 +77,36 @@ def _make_datapipe( ) -> IterDataPipe[Dict[str, Any]] : images_dp, targets_dp = resource_dps - + print(config.split) if config.split == "train": targets_dp = Filter(targets_dp, path_comparator("name", "cars_train_annos.mat")) - - + print("\n"*3) + print("images dp : ",images_dp) + print( "targets_dp :",targets_dp) + print("\n"*3) #right now our both dps are of .tgz and .mat format; for all configs(train and test.) - targets_dp = Mapper(targets_dp,self._read_labels) dp =Zipper(images_dp,targets_dp) dp=hint_sharding(dp) dp=hint_shuffling(dp) + return Mapper(dp,self._read_images_and_labels) - return Mapper(dp,self._collate_and_decode) - - def _collate_and_decode(self, data: Tuple[Any, Any]) -> Dict[str, Any]: - image, target = data # They're both numpy arrays at this point + def _read_images_and_labels(self,data): + image, target = data image_path, image_buffer = image + labels_path, labels_buffer = target image = EncodedImage.from_file(image_buffer) - - - def _read_labels(self,data): - labels = data[0] - labels = read_mat(labels,squeeze_me=True) - labels=labels["annotations"] - print("\n"*10) - print(len(labels["class"])) - bboxs_and_class = zip(labels["bbox_x1"],labels["bbox_y1"],labels["bbox_x2"],labels["bbox_y2"],labels["class"]) - - return bboxs_and_class + labels = read_mat(labels_buffer,squeeze_me=True)["annotations"] + index=image_path[-9:-4] + index = int(image_path[-9:-4]) -1 + + return dict( + index=index, + image_path = image_path, + image = image, + labels_path=labels_path, + classification_label = labels["class"][index] -1, #remember in stanford cars; this labeling starts from 1; hence subtracting 1 + bounding_box = BoundingBox([labels["bbox_x1"][index] ,labels["bbox_y1"][index],labels["bbox_x2"][index] ,labels["bbox_y2"][index]], format="xyxy", image_size=image.image_size) + ) \ No newline at end of file diff --git a/trial.py b/trial.py deleted file mode 100644 index bc0deecf237..00000000000 --- a/trial.py +++ /dev/null @@ -1,25 +0,0 @@ -#from lib2to3.pgen2.token import LSQB -#from tkinter import Label -from distutils.command.config import config -from torchvision.prototype import datasets -#from torchvision.prototype.datasets._builtin.stanford_cars import StanfordCars -dataset= datasets.load("stanford_cars") -ninja2 =0 -#print(type(dataset)) -print(dataset, "jojo") -#print(label) -#print(dat -#print(len(dataset[0])) -for sample in dataset: - #print(len(sample) - print(sample,"is a saple00") - #print(ninja,"hehe") - #print(c,d,e) - ninja2+=1 - if ninja2>10: - break -# print(sample) -# print(label)# this is the content of an item in datapipe returned by _make_datapipe() -# if ninja>29: -# break -# ninja+=2 \ No newline at end of file From d9656221b1635be920f1256e2224c187c00fd561 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 19 Feb 2022 00:57:09 +0530 Subject: [PATCH 06/53] formatting --- .../prototype/datasets/_builtin/__init__.py | 4 +- .../datasets/_builtin/stanford_cars.py | 103 +++++++++--------- trial.py | 25 +++++ 3 files changed, 78 insertions(+), 54 deletions(-) create mode 100644 trial.py diff --git a/torchvision/prototype/datasets/_builtin/__init__.py b/torchvision/prototype/datasets/_builtin/__init__.py index a6a647dd042..160e67c8776 100644 --- a/torchvision/prototype/datasets/_builtin/__init__.py +++ b/torchvision/prototype/datasets/_builtin/__init__.py @@ -8,11 +8,11 @@ from .fer2013 import FER2013 from .gtsrb import GTSRB from .imagenet import ImageNet -from .mnist import MNIST, FashionMNIST, KMNIST, EMNIST, QMNIST +from .mnist import EMNIST, FashionMNIST, KMNIST, MNIST, QMNIST from .oxford_iiit_pet import OxfordIITPet from .pcam import PCAM from .sbd import SBD from .semeion import SEMEION +from .stanford_cars import StanfordCars from .svhn import SVHN from .voc import VOC -from .stanford_cars import StanfordCars diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 46a283b3032..59f59964102 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,57 +1,47 @@ - -from cProfile import label -from hashlib import sha256 import io +import pathlib from collections import namedtuple +from cProfile import label +from hashlib import sha256 from pickletools import read_string1 -from typing import Any, Callable, Dict, List, Optional, Tuple, Iterator -import pathlib +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple from unicodedata import category -from torchdata.datapipes.iter import ( - IterDataPipe, - Mapper, - Filter, - ) + import numpy as np -from torchvision.prototype.datasets.utils._internal import read_mat,hint_sharding, hint_shuffling, path_comparator import torch -from torchdata.datapipes.iter import IterDataPipe, Mapper, Zipper +from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper from torchvision.prototype import features -from torchvision.prototype.features import Label, BoundingBox, _Feature, EncodedImage from torchvision.prototype.datasets.utils import ( Dataset, DatasetConfig, DatasetInfo, + # DatasetType, + HttpResource, OnlineResource, - #DatasetType, - HttpResource ) - -from torchvision.prototype.datasets.utils._internal import ( - hint_sharding, - hint_shuffling, -) -from torchvision.prototype.features import Label +from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat +from torchvision.prototype.features import _Feature, BoundingBox, EncodedImage, Label class StanfordCars(Dataset): def _make_info(self) -> DatasetInfo: return DatasetInfo( name="stanford_cars", - #type=DatasetType.IMAGE, + # type=DatasetType.IMAGE, homepage="https://ai.stanford.edu/~jkrause/cars/car_dataset.html", dependencies=("scipy",), - valid_options=dict(split=("test","train"), - ), + valid_options=dict( + split=("test", "train"), + ), ) _URL_ROOT = "https://ai.stanford.edu/~jkrause/" _URLS = { - "train":f"{_URL_ROOT}car196/cars_train.tgz", - "test":f"{_URL_ROOT}car196/cars_test.tgz", - "test_ground_truth":f"{_URL_ROOT}car196/cars_test_annos_withlabels.mat", - "devkit":f"{_URL_ROOT}cars/car_devkit.tgz" + "train": f"{_URL_ROOT}car196/cars_train.tgz", + "test": f"{_URL_ROOT}car196/cars_test.tgz", + "test_ground_truth": f"{_URL_ROOT}car196/cars_test_annos_withlabels.mat", + "devkit": f"{_URL_ROOT}cars/car_devkit.tgz", } _CHECKSUM = { @@ -62,11 +52,11 @@ def _make_info(self) -> DatasetInfo: } def resources(self, config: DatasetConfig) -> List[OnlineResource]: - resources = [HttpResource(self._URLS[config.split] , sha256=self._CHECKSUM[config.split])] + resources = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])] if config.split == "test": - resources.append(HttpResource(self._URLS["test_ground_truth"],sha256=self._CHECKSUM["test_ground_truth"])) + resources.append(HttpResource(self._URLS["test_ground_truth"], sha256=self._CHECKSUM["test_ground_truth"])) else: - resources.append(HttpResource(url=self._URLS["devkit"],sha256=self._CHECKSUM["devkit"])) + resources.append(HttpResource(url=self._URLS["devkit"], sha256=self._CHECKSUM["devkit"])) return resources def _make_datapipe( @@ -74,39 +64,48 @@ def _make_datapipe( resource_dps: List[IterDataPipe], *, config: DatasetConfig, - ) -> IterDataPipe[Dict[str, Any]] : + ) -> IterDataPipe[Dict[str, Any]]: images_dp, targets_dp = resource_dps print(config.split) if config.split == "train": targets_dp = Filter(targets_dp, path_comparator("name", "cars_train_annos.mat")) - print("\n"*3) - print("images dp : ",images_dp) - print( "targets_dp :",targets_dp) - print("\n"*3) - #right now our both dps are of .tgz and .mat format; for all configs(train and test.) - dp =Zipper(images_dp,targets_dp) - dp=hint_sharding(dp) - dp=hint_shuffling(dp) - return Mapper(dp,self._read_images_and_labels) - + print("\n" * 3) + print("images dp : ", images_dp) + print("targets_dp :", targets_dp) + print("\n" * 3) + # right now our both dps are of .tgz and .mat format; for all configs(train and test.) + dp = Zipper(images_dp, targets_dp) + dp = hint_sharding(dp) + dp = hint_shuffling(dp) + return Mapper(dp, self._read_images_and_labels) - def _read_images_and_labels(self,data): - image, target = data + def _read_images_and_labels(self, data): + image, target = data image_path, image_buffer = image labels_path, labels_buffer = target image = EncodedImage.from_file(image_buffer) - labels = read_mat(labels_buffer,squeeze_me=True)["annotations"] - index=image_path[-9:-4] - index = int(image_path[-9:-4]) -1 + labels = read_mat(labels_buffer, squeeze_me=True)["annotations"] + index = image_path[-9:-4] + index = int(image_path[-9:-4]) - 1 return dict( index=index, - image_path = image_path, - image = image, + image_path=image_path, + image=image, labels_path=labels_path, - classification_label = labels["class"][index] -1, #remember in stanford cars; this labeling starts from 1; hence subtracting 1 - bounding_box = BoundingBox([labels["bbox_x1"][index] ,labels["bbox_y1"][index],labels["bbox_x2"][index] ,labels["bbox_y2"][index]], format="xyxy", image_size=image.image_size) - ) \ No newline at end of file + classification_label=labels["class"][index] + - 1, # remember in stanford cars; this labeling starts from 1; hence subtracting 1 + bounding_box=BoundingBox( + [ + labels["bbox_x1"][index], + labels["bbox_y1"][index], + labels["bbox_x2"][index], + labels["bbox_y2"][index], + ], + format="xyxy", + image_size=image.image_size, + ), + ) diff --git a/trial.py b/trial.py new file mode 100644 index 00000000000..bc0deecf237 --- /dev/null +++ b/trial.py @@ -0,0 +1,25 @@ +#from lib2to3.pgen2.token import LSQB +#from tkinter import Label +from distutils.command.config import config +from torchvision.prototype import datasets +#from torchvision.prototype.datasets._builtin.stanford_cars import StanfordCars +dataset= datasets.load("stanford_cars") +ninja2 =0 +#print(type(dataset)) +print(dataset, "jojo") +#print(label) +#print(dat +#print(len(dataset[0])) +for sample in dataset: + #print(len(sample) + print(sample,"is a saple00") + #print(ninja,"hehe") + #print(c,d,e) + ninja2+=1 + if ninja2>10: + break +# print(sample) +# print(label)# this is the content of an item in datapipe returned by _make_datapipe() +# if ninja>29: +# break +# ninja+=2 \ No newline at end of file From eb2bc93b54393ceeefd13f7cf0d7a1ec4d7ae973 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 19 Feb 2022 00:57:28 +0530 Subject: [PATCH 07/53] formatting --- trial.py | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 trial.py diff --git a/trial.py b/trial.py deleted file mode 100644 index bc0deecf237..00000000000 --- a/trial.py +++ /dev/null @@ -1,25 +0,0 @@ -#from lib2to3.pgen2.token import LSQB -#from tkinter import Label -from distutils.command.config import config -from torchvision.prototype import datasets -#from torchvision.prototype.datasets._builtin.stanford_cars import StanfordCars -dataset= datasets.load("stanford_cars") -ninja2 =0 -#print(type(dataset)) -print(dataset, "jojo") -#print(label) -#print(dat -#print(len(dataset[0])) -for sample in dataset: - #print(len(sample) - print(sample,"is a saple00") - #print(ninja,"hehe") - #print(c,d,e) - ninja2+=1 - if ninja2>10: - break -# print(sample) -# print(label)# this is the content of an item in datapipe returned by _make_datapipe() -# if ninja>29: -# break -# ninja+=2 \ No newline at end of file From a4041a89c0d5d1fe4fe85316e46de6c4ddc081b5 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 19 Feb 2022 01:02:19 +0530 Subject: [PATCH 08/53] formatting --- .../datasets/_builtin/stanford_cars.py | 103 +++++++++--------- 1 file changed, 50 insertions(+), 53 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 59f59964102..d43cf944908 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,47 +1,53 @@ -import io -import pathlib -from collections import namedtuple + from cProfile import label from hashlib import sha256 -from pickletools import read_string1 -from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple -from unicodedata import category - +import io +from typing import Any, Callable, Optional, Tuple, Iterator +from torchdata.datapipes.iter import ( + IterDataPipe, + Mapper, + Filter, + ) import numpy as np +from torchvision.prototype.datasets.utils._internal import read_mat,hint_sharding, hint_shuffling, path_comparator import torch -from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper +from torchdata.datapipes.iter import IterDataPipe, Mapper, Zipper from torchvision.prototype import features +from torchvision.prototype.features import Label, BoundingBox, _Feature, EncodedImage from torchvision.prototype.datasets.utils import ( Dataset, DatasetConfig, DatasetInfo, - # DatasetType, - HttpResource, OnlineResource, + #DatasetType, + HttpResource ) -from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat -from torchvision.prototype.features import _Feature, BoundingBox, EncodedImage, Label + +from torchvision.prototype.datasets.utils._internal import ( + hint_sharding, + hint_shuffling, +) +from torchvision.prototype.features import Label class StanfordCars(Dataset): def _make_info(self) -> DatasetInfo: return DatasetInfo( name="stanford_cars", - # type=DatasetType.IMAGE, + #type=DatasetType.IMAGE, homepage="https://ai.stanford.edu/~jkrause/cars/car_dataset.html", dependencies=("scipy",), - valid_options=dict( - split=("test", "train"), - ), + valid_options=dict(split=("test","train"), + ), ) _URL_ROOT = "https://ai.stanford.edu/~jkrause/" _URLS = { - "train": f"{_URL_ROOT}car196/cars_train.tgz", - "test": f"{_URL_ROOT}car196/cars_test.tgz", - "test_ground_truth": f"{_URL_ROOT}car196/cars_test_annos_withlabels.mat", - "devkit": f"{_URL_ROOT}cars/car_devkit.tgz", + "train":f"{_URL_ROOT}car196/cars_train.tgz", + "test":f"{_URL_ROOT}car196/cars_test.tgz", + "test_ground_truth":f"{_URL_ROOT}car196/cars_test_annos_withlabels.mat", + "devkit":f"{_URL_ROOT}cars/car_devkit.tgz" } _CHECKSUM = { @@ -52,11 +58,11 @@ def _make_info(self) -> DatasetInfo: } def resources(self, config: DatasetConfig) -> List[OnlineResource]: - resources = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])] + resources = [HttpResource(self._URLS[config.split] , sha256=self._CHECKSUM[config.split])] if config.split == "test": - resources.append(HttpResource(self._URLS["test_ground_truth"], sha256=self._CHECKSUM["test_ground_truth"])) + resources.append(HttpResource(self._URLS["test_ground_truth"],sha256=self._CHECKSUM["test_ground_truth"])) else: - resources.append(HttpResource(url=self._URLS["devkit"], sha256=self._CHECKSUM["devkit"])) + resources.append(HttpResource(url=self._URLS["devkit"],sha256=self._CHECKSUM["devkit"])) return resources def _make_datapipe( @@ -64,48 +70,39 @@ def _make_datapipe( resource_dps: List[IterDataPipe], *, config: DatasetConfig, - ) -> IterDataPipe[Dict[str, Any]]: + ) -> IterDataPipe[Dict[str, Any]] : images_dp, targets_dp = resource_dps print(config.split) if config.split == "train": targets_dp = Filter(targets_dp, path_comparator("name", "cars_train_annos.mat")) - print("\n" * 3) - print("images dp : ", images_dp) - print("targets_dp :", targets_dp) - print("\n" * 3) - # right now our both dps are of .tgz and .mat format; for all configs(train and test.) - dp = Zipper(images_dp, targets_dp) - dp = hint_sharding(dp) - dp = hint_shuffling(dp) - return Mapper(dp, self._read_images_and_labels) + print("\n"*3) + print("images dp : ",images_dp) + print( "targets_dp :",targets_dp) + print("\n"*3) + #right now our both dps are of .tgz and .mat format; for all configs(train and test.) + dp =Zipper(images_dp,targets_dp) + dp=hint_sharding(dp) + dp=hint_shuffling(dp) + return Mapper(dp,self._read_images_and_labels) - def _read_images_and_labels(self, data): - image, target = data + + def _read_images_and_labels(self,data): + image, target = data image_path, image_buffer = image labels_path, labels_buffer = target image = EncodedImage.from_file(image_buffer) - labels = read_mat(labels_buffer, squeeze_me=True)["annotations"] - index = image_path[-9:-4] - index = int(image_path[-9:-4]) - 1 + labels = read_mat(labels_buffer,squeeze_me=True)["annotations"] + index=image_path[-9:-4] + index = int(image_path[-9:-4]) -1 return dict( index=index, - image_path=image_path, - image=image, + image_path = image_path, + image = image, labels_path=labels_path, - classification_label=labels["class"][index] - - 1, # remember in stanford cars; this labeling starts from 1; hence subtracting 1 - bounding_box=BoundingBox( - [ - labels["bbox_x1"][index], - labels["bbox_y1"][index], - labels["bbox_x2"][index], - labels["bbox_y2"][index], - ], - format="xyxy", - image_size=image.image_size, - ), - ) + classification_label = labels["class"][index] -1, #remember in stanford cars; this labeling starts from 1; hence subtracting 1 + bounding_box = BoundingBox([labels["bbox_x1"][index] ,labels["bbox_y1"][index],labels["bbox_x2"][index] ,labels["bbox_y2"][index]], format="xyxy", image_size=image.image_size) + ) \ No newline at end of file From c681d399252079507adb5870da3762a9f6668122 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 19 Feb 2022 10:43:23 +0530 Subject: [PATCH 09/53] formatting --- .../datasets/_builtin/stanford_cars.py | 75 +++++++------------ 1 file changed, 28 insertions(+), 47 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index d43cf944908..d4dbdf654aa 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,53 +1,41 @@ - -from cProfile import label -from hashlib import sha256 -import io -from typing import Any, Callable, Optional, Tuple, Iterator +from typing import Any, List, Dict from torchdata.datapipes.iter import ( IterDataPipe, Mapper, Filter, - ) -import numpy as np -from torchvision.prototype.datasets.utils._internal import read_mat,hint_sharding, hint_shuffling, path_comparator -import torch -from torchdata.datapipes.iter import IterDataPipe, Mapper, Zipper -from torchvision.prototype import features -from torchvision.prototype.features import Label, BoundingBox, _Feature, EncodedImage + Zipper, +) +from torchvision.prototype.features import BoundingBox, EncodedImage from torchvision.prototype.datasets.utils import ( Dataset, DatasetConfig, DatasetInfo, OnlineResource, - #DatasetType, HttpResource ) - - from torchvision.prototype.datasets.utils._internal import ( hint_sharding, hint_shuffling, + read_mat, + path_comparator ) -from torchvision.prototype.features import Label class StanfordCars(Dataset): def _make_info(self) -> DatasetInfo: return DatasetInfo( name="stanford_cars", - #type=DatasetType.IMAGE, homepage="https://ai.stanford.edu/~jkrause/cars/car_dataset.html", dependencies=("scipy",), - valid_options=dict(split=("test","train"), - ), + valid_options=dict(split=("test", "train"),), ) _URL_ROOT = "https://ai.stanford.edu/~jkrause/" _URLS = { - "train":f"{_URL_ROOT}car196/cars_train.tgz", - "test":f"{_URL_ROOT}car196/cars_test.tgz", - "test_ground_truth":f"{_URL_ROOT}car196/cars_test_annos_withlabels.mat", - "devkit":f"{_URL_ROOT}cars/car_devkit.tgz" + "train": f"{_URL_ROOT}car196/cars_train.tgz", + "test": f"{_URL_ROOT}car196/cars_test.tgz", + "test_ground_truth": f"{_URL_ROOT}car196/cars_test_annos_withlabels.mat", + "devkit": f"{_URL_ROOT}cars/car_devkit.tgz", } _CHECKSUM = { @@ -58,11 +46,11 @@ def _make_info(self) -> DatasetInfo: } def resources(self, config: DatasetConfig) -> List[OnlineResource]: - resources = [HttpResource(self._URLS[config.split] , sha256=self._CHECKSUM[config.split])] + resources = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])] if config.split == "test": - resources.append(HttpResource(self._URLS["test_ground_truth"],sha256=self._CHECKSUM["test_ground_truth"])) + resources.append(HttpResource(self._URLS["test_ground_truth"], sha256=self._CHECKSUM["test_ground_truth"])) else: - resources.append(HttpResource(url=self._URLS["devkit"],sha256=self._CHECKSUM["devkit"])) + resources.append(HttpResource(url=self._URLS["devkit"], sha256=self._CHECKSUM["devkit"])) return resources def _make_datapipe( @@ -76,33 +64,26 @@ def _make_datapipe( print(config.split) if config.split == "train": targets_dp = Filter(targets_dp, path_comparator("name", "cars_train_annos.mat")) - print("\n"*3) - print("images dp : ",images_dp) - print( "targets_dp :",targets_dp) - print("\n"*3) - #right now our both dps are of .tgz and .mat format; for all configs(train and test.) - dp =Zipper(images_dp,targets_dp) - dp=hint_sharding(dp) - dp=hint_shuffling(dp) - return Mapper(dp,self._read_images_and_labels) - - - def _read_images_and_labels(self,data): - image, target = data + dp = Zipper(images_dp, targets_dp) + dp = hint_sharding(dp) + dp = hint_shuffling(dp) + return Mapper(dp, self._read_images_and_labels) + def _read_images_and_labels(self, data): + image, target = data image_path, image_buffer = image labels_path, labels_buffer = target image = EncodedImage.from_file(image_buffer) - labels = read_mat(labels_buffer,squeeze_me=True)["annotations"] - index=image_path[-9:-4] - index = int(image_path[-9:-4]) -1 + labels = read_mat(labels_buffer, squeeze_me=True)["annotations"] + index = image_path[-9:-4] + index = int(image_path[-9:-4]) - 1 return dict( index=index, - image_path = image_path, - image = image, + image_path=image_path, + image=image, labels_path=labels_path, - classification_label = labels["class"][index] -1, #remember in stanford cars; this labeling starts from 1; hence subtracting 1 - bounding_box = BoundingBox([labels["bbox_x1"][index] ,labels["bbox_y1"][index],labels["bbox_x2"][index] ,labels["bbox_y2"][index]], format="xyxy", image_size=image.image_size) - ) \ No newline at end of file + classification_label=labels["class"][index] - 1, + bounding_box=BoundingBox([labels["bbox_x1"][index], labels["bbox_y1"][index], labels["bbox_x2"][index], labels["bbox_y2"][index]], format="xyxy", image_size=image.image_size) + ) From a3953fe6b3291d4105f8587de9836dc72eb84ae4 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 19 Feb 2022 10:46:49 +0530 Subject: [PATCH 10/53] formatting --- .../datasets/_builtin/stanford_cars.py | 42 +++++++++---------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index d4dbdf654aa..e1911ba8a20 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,24 +1,9 @@ -from typing import Any, List, Dict -from torchdata.datapipes.iter import ( - IterDataPipe, - Mapper, - Filter, - Zipper, -) +from typing import Any, Dict, List + +from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper +from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource +from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat from torchvision.prototype.features import BoundingBox, EncodedImage -from torchvision.prototype.datasets.utils import ( - Dataset, - DatasetConfig, - DatasetInfo, - OnlineResource, - HttpResource -) -from torchvision.prototype.datasets.utils._internal import ( - hint_sharding, - hint_shuffling, - read_mat, - path_comparator -) class StanfordCars(Dataset): @@ -27,7 +12,9 @@ def _make_info(self) -> DatasetInfo: name="stanford_cars", homepage="https://ai.stanford.edu/~jkrause/cars/car_dataset.html", dependencies=("scipy",), - valid_options=dict(split=("test", "train"),), + valid_options=dict( + split=("test", "train"), + ), ) _URL_ROOT = "https://ai.stanford.edu/~jkrause/" @@ -58,7 +45,7 @@ def _make_datapipe( resource_dps: List[IterDataPipe], *, config: DatasetConfig, - ) -> IterDataPipe[Dict[str, Any]] : + ) -> IterDataPipe[Dict[str, Any]]: images_dp, targets_dp = resource_dps print(config.split) @@ -85,5 +72,14 @@ def _read_images_and_labels(self, data): image=image, labels_path=labels_path, classification_label=labels["class"][index] - 1, - bounding_box=BoundingBox([labels["bbox_x1"][index], labels["bbox_y1"][index], labels["bbox_x2"][index], labels["bbox_y2"][index]], format="xyxy", image_size=image.image_size) + bounding_box=BoundingBox( + [ + labels["bbox_x1"][index], + labels["bbox_y1"][index], + labels["bbox_x2"][index], + labels["bbox_y2"][index], + ], + format="xyxy", + image_size=image.image_size, + ), ) From 8bd73483242896d7fa7611faa29d27c4830dd1a8 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sun, 27 Feb 2022 20:52:36 +0530 Subject: [PATCH 11/53] minor changes --- ...rs.categories => stanford-cars.categories} | 0 .../datasets/_builtin/stanford_cars.py | 43 ++++++++++--------- 2 files changed, 23 insertions(+), 20 deletions(-) rename torchvision/prototype/datasets/_builtin/{stanford_cars.categories => stanford-cars.categories} (100%) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.categories b/torchvision/prototype/datasets/_builtin/stanford-cars.categories similarity index 100% rename from torchvision/prototype/datasets/_builtin/stanford_cars.categories rename to torchvision/prototype/datasets/_builtin/stanford-cars.categories diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index e1911ba8a20..c77dee30845 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,15 +1,15 @@ from typing import Any, Dict, List -from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper +from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper, IterKeyZipper from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource -from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat +from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat, getitem from torchvision.prototype.features import BoundingBox, EncodedImage class StanfordCars(Dataset): def _make_info(self) -> DatasetInfo: return DatasetInfo( - name="stanford_cars", + name="stanford-cars", homepage="https://ai.stanford.edu/~jkrause/cars/car_dataset.html", dependencies=("scipy",), valid_options=dict( @@ -40,23 +40,8 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: resources.append(HttpResource(url=self._URLS["devkit"], sha256=self._CHECKSUM["devkit"])) return resources - def _make_datapipe( - self, - resource_dps: List[IterDataPipe], - *, - config: DatasetConfig, - ) -> IterDataPipe[Dict[str, Any]]: - - images_dp, targets_dp = resource_dps - print(config.split) - if config.split == "train": - targets_dp = Filter(targets_dp, path_comparator("name", "cars_train_annos.mat")) - dp = Zipper(images_dp, targets_dp) - dp = hint_sharding(dp) - dp = hint_shuffling(dp) - return Mapper(dp, self._read_images_and_labels) - - def _read_images_and_labels(self, data): + + def _prepare_sample(self, data): image, target = data image_path, image_buffer = image labels_path, labels_buffer = target @@ -83,3 +68,21 @@ def _read_images_and_labels(self, data): image_size=image.image_size, ), ) + + + def _make_datapipe( + self, + resource_dps: List[IterDataPipe], + *, + config: DatasetConfig, + ) -> IterDataPipe[Dict[str, Any]]: + + images_dp, targets_dp = resource_dps + if config.split == "train": + targets_dp = Filter(targets_dp, path_comparator("name", "cars_train_annos.mat")) + + dp = Zipper(images_dp, targets_dp) + dp = hint_sharding(dp) + dp = hint_shuffling(dp) + + return Mapper(dp, self._prepare_sample) From 7ad896eb9350df972a28b6b695aaae18351d93e2 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sun, 27 Feb 2022 20:53:27 +0530 Subject: [PATCH 12/53] minor changes --- torchvision/prototype/datasets/_builtin/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/prototype/datasets/_builtin/__init__.py b/torchvision/prototype/datasets/_builtin/__init__.py index 160e67c8776..2396287d77f 100644 --- a/torchvision/prototype/datasets/_builtin/__init__.py +++ b/torchvision/prototype/datasets/_builtin/__init__.py @@ -8,7 +8,7 @@ from .fer2013 import FER2013 from .gtsrb import GTSRB from .imagenet import ImageNet -from .mnist import EMNIST, FashionMNIST, KMNIST, MNIST, QMNIST +from .mnist import MNIST, FashionMNIST, KMNIST, EMNIST, QMNIST from .oxford_iiit_pet import OxfordIITPet from .pcam import PCAM from .sbd import SBD From 5922e1b30c4ccafab1793ff2a8225e88db79f644 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Thu, 3 Mar 2022 15:01:20 +0530 Subject: [PATCH 13/53] issue resolved --- .../datasets/_builtin/stanford_cars.py | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index c77dee30845..c9465f1bdb9 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,11 +1,23 @@ -from typing import Any, Dict, List - +from typing import Any, Dict, List,Tuple,Iterator +import numpy as np from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper, IterKeyZipper from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat, getitem from torchvision.prototype.features import BoundingBox, EncodedImage +class _StanfordLabelReader(IterDataPipe[Tuple[np.ndarray, int]]): + def __init__(self, datapipe: IterDataPipe[Dict[str, Any]]) -> None: + self.datapipe = datapipe + + def __iter__(self) -> Iterator[Tuple[str, Dict[str, str]]]: + for _, file in self.datapipe: + file = iter(read_mat(file,squeeze_me=True)["annotations"]) + for line in file: + yield line + + + class StanfordCars(Dataset): def _make_info(self) -> DatasetInfo: return DatasetInfo( @@ -26,10 +38,10 @@ def _make_info(self) -> DatasetInfo: } _CHECKSUM = { - "train": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", + "train": "b97deb463af7d58b6bfaa18b2a4de9829f0f79e8ce663dfa9261bf7810e9accd", "test": "bffea656d6f425cba3c91c6d83336e4c5f86c6cffd8975b0f375d3a10da8e243", - "test_ground_truth": "790f75be8ea34eeded134cc559332baf23e30e91367e9ddca97d26ed9b895f05", - "devkit": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", + "": "790f75be8ea34eeded134cc559332baf23e30e91367e9ddca97d26ed9b895f05", + "": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", } def resources(self, config: DatasetConfig) -> List[OnlineResource]: @@ -44,26 +56,24 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: def _prepare_sample(self, data): image, target = data image_path, image_buffer = image - labels_path, labels_buffer = target - image = EncodedImage.from_file(image_buffer) - labels = read_mat(labels_buffer, squeeze_me=True)["annotations"] index = image_path[-9:-4] index = int(image_path[-9:-4]) - 1 + return dict( index=index, image_path=image_path, + index1=target[-1], image=image, - labels_path=labels_path, - classification_label=labels["class"][index] - 1, + classification_label=target[4] - 1, bounding_box=BoundingBox( [ - labels["bbox_x1"][index], - labels["bbox_y1"][index], - labels["bbox_x2"][index], - labels["bbox_y2"][index], - ], + target[0], + target[1], + target[2], + target[3], + ], format="xyxy", image_size=image.image_size, ), @@ -80,7 +90,8 @@ def _make_datapipe( images_dp, targets_dp = resource_dps if config.split == "train": targets_dp = Filter(targets_dp, path_comparator("name", "cars_train_annos.mat")) - + print(images_dp) + targets_dp = _StanfordLabelReader(targets_dp) dp = Zipper(images_dp, targets_dp) dp = hint_sharding(dp) dp = hint_shuffling(dp) From bca2a6b997f88228cc9e19dd1cc395e99cc95bc9 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Thu, 3 Mar 2022 15:14:25 +0530 Subject: [PATCH 14/53] both datapipes are in sync --- .../prototype/datasets/_builtin/stanford_cars.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index c9465f1bdb9..e3ca74c1af0 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,12 +1,12 @@ from typing import Any, Dict, List,Tuple,Iterator import numpy as np -from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper, IterKeyZipper +from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource -from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat, getitem +from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat from torchvision.prototype.features import BoundingBox, EncodedImage -class _StanfordLabelReader(IterDataPipe[Tuple[np.ndarray, int]]): +class _StanfordCarsLabelReader(IterDataPipe[Tuple[np.ndarray, int]]): def __init__(self, datapipe: IterDataPipe[Dict[str, Any]]) -> None: self.datapipe = datapipe @@ -40,8 +40,8 @@ def _make_info(self) -> DatasetInfo: _CHECKSUM = { "train": "b97deb463af7d58b6bfaa18b2a4de9829f0f79e8ce663dfa9261bf7810e9accd", "test": "bffea656d6f425cba3c91c6d83336e4c5f86c6cffd8975b0f375d3a10da8e243", - "": "790f75be8ea34eeded134cc559332baf23e30e91367e9ddca97d26ed9b895f05", - "": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", + "test_ground_truth": "790f75be8ea34eeded134cc559332baf23e30e91367e9ddca97d26ed9b895f05", + "devkit": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", } def resources(self, config: DatasetConfig) -> List[OnlineResource]: @@ -50,6 +50,7 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: resources.append(HttpResource(self._URLS["test_ground_truth"], sha256=self._CHECKSUM["test_ground_truth"])) else: resources.append(HttpResource(url=self._URLS["devkit"], sha256=self._CHECKSUM["devkit"])) + return resources @@ -90,8 +91,7 @@ def _make_datapipe( images_dp, targets_dp = resource_dps if config.split == "train": targets_dp = Filter(targets_dp, path_comparator("name", "cars_train_annos.mat")) - print(images_dp) - targets_dp = _StanfordLabelReader(targets_dp) + targets_dp = _StanfordCarsLabelReader(targets_dp) dp = Zipper(images_dp, targets_dp) dp = hint_sharding(dp) dp = hint_shuffling(dp) From 7d0b908fb907d970bca9ce4b081f77feee48c0c9 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Thu, 3 Mar 2022 15:21:12 +0530 Subject: [PATCH 15/53] checksum dict keys changed --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index e3ca74c1af0..f17c7d14fb8 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -40,16 +40,16 @@ def _make_info(self) -> DatasetInfo: _CHECKSUM = { "train": "b97deb463af7d58b6bfaa18b2a4de9829f0f79e8ce663dfa9261bf7810e9accd", "test": "bffea656d6f425cba3c91c6d83336e4c5f86c6cffd8975b0f375d3a10da8e243", - "test_ground_truth": "790f75be8ea34eeded134cc559332baf23e30e91367e9ddca97d26ed9b895f05", - "devkit": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", + "cars_test_annos_withlabels.mat": "790f75be8ea34eeded134cc559332baf23e30e91367e9ddca97d26ed9b895f05", + "car_devkit": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", } def resources(self, config: DatasetConfig) -> List[OnlineResource]: resources = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])] if config.split == "test": - resources.append(HttpResource(self._URLS["test_ground_truth"], sha256=self._CHECKSUM["test_ground_truth"])) + resources.append(HttpResource(self._URLS["cars_test_annos_withlabels"], sha256=self._CHECKSUM["cars_test_annos_withlabels"])) else: - resources.append(HttpResource(url=self._URLS["devkit"], sha256=self._CHECKSUM["devkit"])) + resources.append(HttpResource(url=self._URLS["car_devkit"], sha256=self._CHECKSUM["car_devkit"])) return resources From 862fc1e444a5450dd25e67dbf745aaa7bdff7552 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Thu, 3 Mar 2022 15:38:19 +0530 Subject: [PATCH 16/53] formatting --- .../prototype/datasets/_builtin/stanford_cars.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index f17c7d14fb8..0247cbe6170 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List,Tuple,Iterator +from typing import Any, Dict, List, Tuple, Iterator import numpy as np from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource @@ -12,12 +12,11 @@ def __init__(self, datapipe: IterDataPipe[Dict[str, Any]]) -> None: def __iter__(self) -> Iterator[Tuple[str, Dict[str, str]]]: for _, file in self.datapipe: - file = iter(read_mat(file,squeeze_me=True)["annotations"]) + file = iter(read_mat(file, squeeze_me=True)["annotations"]) for line in file: yield line - class StanfordCars(Dataset): def _make_info(self) -> DatasetInfo: return DatasetInfo( @@ -50,17 +49,14 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: resources.append(HttpResource(self._URLS["cars_test_annos_withlabels"], sha256=self._CHECKSUM["cars_test_annos_withlabels"])) else: resources.append(HttpResource(url=self._URLS["car_devkit"], sha256=self._CHECKSUM["car_devkit"])) - - return resources + return resources - def _prepare_sample(self, data): image, target = data image_path, image_buffer = image image = EncodedImage.from_file(image_buffer) index = image_path[-9:-4] index = int(image_path[-9:-4]) - 1 - return dict( index=index, @@ -74,13 +70,12 @@ def _prepare_sample(self, data): target[1], target[2], target[3], - ], + ], format="xyxy", image_size=image.image_size, ), ) - def _make_datapipe( self, resource_dps: List[IterDataPipe], @@ -95,5 +90,4 @@ def _make_datapipe( dp = Zipper(images_dp, targets_dp) dp = hint_sharding(dp) dp = hint_shuffling(dp) - return Mapper(dp, self._prepare_sample) From 8f91280ffad0898cbd7915f048bd36f88133142b Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Thu, 3 Mar 2022 16:37:04 +0530 Subject: [PATCH 17/53] "ufmt formatting" --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 0247cbe6170..c3787c7b50e 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,4 +1,5 @@ from typing import Any, Dict, List, Tuple, Iterator + import numpy as np from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource @@ -46,7 +47,11 @@ def _make_info(self) -> DatasetInfo: def resources(self, config: DatasetConfig) -> List[OnlineResource]: resources = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])] if config.split == "test": - resources.append(HttpResource(self._URLS["cars_test_annos_withlabels"], sha256=self._CHECKSUM["cars_test_annos_withlabels"])) + resources.append( + HttpResource( + self._URLS["cars_test_annos_withlabels"], sha256=self._CHECKSUM["cars_test_annos_withlabels"] + ) + ) else: resources.append(HttpResource(url=self._URLS["car_devkit"], sha256=self._CHECKSUM["car_devkit"])) return resources From 873a6f87551eae8c2a522a6825159280aca4c318 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Thu, 3 Mar 2022 16:37:30 +0530 Subject: [PATCH 18/53] "ufmt formatting" --- torchvision/prototype/datasets/_builtin/celeba.py | 10 +++++----- trial.py | 9 +++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 trial.py diff --git a/torchvision/prototype/datasets/_builtin/celeba.py b/torchvision/prototype/datasets/_builtin/celeba.py index 191f49e9d53..40a823fa497 100644 --- a/torchvision/prototype/datasets/_builtin/celeba.py +++ b/torchvision/prototype/datasets/_builtin/celeba.py @@ -55,7 +55,7 @@ def __iter__(self) -> Iterator[Tuple[str, Dict[str, str]]]: # Some files do not include a label for the image ID column if fieldnames[0] != "image_id": fieldnames.insert(0, "image_id") - + print(file) for line in csv.DictReader(file, fieldnames=fieldnames, dialect="celeba"): yield line.pop("image_id"), line @@ -76,7 +76,7 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: ) images = GDriveResource( "0B7EVK8r0v71pZjFTYXZWM3FlRnM", - sha256="46fb89443c578308acf364d7d379fe1b9efb793042c0af734b6112e4fd3a8c74", + sha256="d1f6c9c2dd5a3c67a15455100a238bacbbfc0c1dd429eb87ff0283a1a7476751", file_name="img_align_celeba.zip", ) identities = GDriveResource( @@ -86,17 +86,17 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: ) attributes = GDriveResource( "0B7EVK8r0v71pblRyaVFSWGxPY0U", - sha256="f0e5da289d5ccf75ffe8811132694922b60f2af59256ed362afa03fefba324d0", + sha256="c70328321c257ff62852a8b2fdd4aca00080a87cd9ae7dca1f3978970e405444", file_name="list_attr_celeba.txt", ) bounding_boxes = GDriveResource( "0B7EVK8r0v71pbThiMVRxWXZ4dU0", - sha256="7487a82e57c4bb956c5445ae2df4a91ffa717e903c5fa22874ede0820c8ec41b", + sha256="1f27aebea6e592bac525951cb13235ecfe96f0213eb5c35367659bf777796b75", file_name="list_bbox_celeba.txt", ) landmarks = GDriveResource( "0B7EVK8r0v71pd0FJY3Blby1HUTQ", - sha256="6c02a87569907f6db2ba99019085697596730e8129f67a3d61659f198c48d43b", + sha256="5e30bd75b4e3e0c9e9d3bc42489f163174e298c93ef3e651d6c4a9a68acf0089", file_name="list_landmarks_align_celeba.txt", ) return [splits, images, identities, attributes, bounding_boxes, landmarks] diff --git a/trial.py b/trial.py new file mode 100644 index 00000000000..21167c32ea3 --- /dev/null +++ b/trial.py @@ -0,0 +1,9 @@ +from torchvision.prototype import datasets + +dataset = datasets.load("stanford-cars") + +for sample in dataset: + if sample["image_path"][-9:] != sample["index1"]: + print("Use IterKeyZipper instead of Zipper") + +# Or you can also inspect the sample in a debugger \ No newline at end of file From 4c40f86ba0f2a280edbb14da9726f87dcc998970 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Thu, 3 Mar 2022 16:39:17 +0530 Subject: [PATCH 19/53] formatting --- trial.py | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 trial.py diff --git a/trial.py b/trial.py deleted file mode 100644 index 21167c32ea3..00000000000 --- a/trial.py +++ /dev/null @@ -1,9 +0,0 @@ -from torchvision.prototype import datasets - -dataset = datasets.load("stanford-cars") - -for sample in dataset: - if sample["image_path"][-9:] != sample["index1"]: - print("Use IterKeyZipper instead of Zipper") - -# Or you can also inspect the sample in a debugger \ No newline at end of file From 752bc4b2c519cbeebd1ae605fb884e82dc7ac319 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Thu, 3 Mar 2022 16:41:05 +0530 Subject: [PATCH 20/53] minor change --- torchvision/prototype/datasets/_builtin/celeba.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/celeba.py b/torchvision/prototype/datasets/_builtin/celeba.py index 40a823fa497..bca54e66d0b 100644 --- a/torchvision/prototype/datasets/_builtin/celeba.py +++ b/torchvision/prototype/datasets/_builtin/celeba.py @@ -55,7 +55,7 @@ def __iter__(self) -> Iterator[Tuple[str, Dict[str, str]]]: # Some files do not include a label for the image ID column if fieldnames[0] != "image_id": fieldnames.insert(0, "image_id") - print(file) + for line in csv.DictReader(file, fieldnames=fieldnames, dialect="celeba"): yield line.pop("image_id"), line @@ -76,7 +76,7 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: ) images = GDriveResource( "0B7EVK8r0v71pZjFTYXZWM3FlRnM", - sha256="d1f6c9c2dd5a3c67a15455100a238bacbbfc0c1dd429eb87ff0283a1a7476751", + sha256="46fb89443c578308acf364d7d379fe1b9efb793042c0af734b6112e4fd3a8c74", file_name="img_align_celeba.zip", ) identities = GDriveResource( @@ -86,17 +86,17 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: ) attributes = GDriveResource( "0B7EVK8r0v71pblRyaVFSWGxPY0U", - sha256="c70328321c257ff62852a8b2fdd4aca00080a87cd9ae7dca1f3978970e405444", + sha256="f0e5da289d5ccf75ffe8811132694922b60f2af59256ed362afa03fefba324d0", file_name="list_attr_celeba.txt", ) bounding_boxes = GDriveResource( "0B7EVK8r0v71pbThiMVRxWXZ4dU0", - sha256="1f27aebea6e592bac525951cb13235ecfe96f0213eb5c35367659bf777796b75", + sha256="7487a82e57c4bb956c5445ae2df4a91ffa717e903c5fa22874ede0820c8ec41b", file_name="list_bbox_celeba.txt", ) landmarks = GDriveResource( "0B7EVK8r0v71pd0FJY3Blby1HUTQ", - sha256="5e30bd75b4e3e0c9e9d3bc42489f163174e298c93ef3e651d6c4a9a68acf0089", + sha256="6c02a87569907f6db2ba99019085697596730e8129f67a3d61659f198c48d43b", file_name="list_landmarks_align_celeba.txt", ) return [splits, images, identities, attributes, bounding_boxes, landmarks] @@ -185,4 +185,4 @@ def _make_datapipe( ref_key_fn=getitem(0, 0), buffer_size=INFINITE_BUFFER_SIZE, ) - return Mapper(dp, self._prepare_sample) + return Mapper(dp, self._prepare_sample) \ No newline at end of file From 2583c7cc13342260074c3a42e4673837ba0bb9eb Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Thu, 3 Mar 2022 16:42:14 +0530 Subject: [PATCH 21/53] minor change --- torchvision/prototype/datasets/_builtin/celeba.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/prototype/datasets/_builtin/celeba.py b/torchvision/prototype/datasets/_builtin/celeba.py index bca54e66d0b..191f49e9d53 100644 --- a/torchvision/prototype/datasets/_builtin/celeba.py +++ b/torchvision/prototype/datasets/_builtin/celeba.py @@ -185,4 +185,4 @@ def _make_datapipe( ref_key_fn=getitem(0, 0), buffer_size=INFINITE_BUFFER_SIZE, ) - return Mapper(dp, self._prepare_sample) \ No newline at end of file + return Mapper(dp, self._prepare_sample) From 197d2e19f4201908be2872e813ea0fa4f97e249f Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 10:19:06 +0530 Subject: [PATCH 22/53] test added --- test/builtin_dataset_mocks.py | 40 +++++++++++++++++++ .../datasets/_builtin/stanford_cars.py | 4 +- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index 94305952568..21af3eba03f 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1390,3 +1390,43 @@ def pcam(info, root, config): compressed_file.write(compressed_data) return num_images + + +@register_mock +def stanford_cars_test(info,root,config): + import scipy.io as io + from np.core.records import fromarrays + + num_examples = {"train": 5, "test": 7}[config["split"]] + num_classes = 3 + base_folder = pathlib.Path(root) / "stanford-cars" + + devkit = base_folder / "devkit" + devkit.mkdir(parents=True) + + if config["split"] == "train": + images_folder_name = "cars_train" + annotations_mat_path = devkit / "cars_train_annos.mat" + else: + images_folder_name = "cars_test" + annotations_mat_path = base_folder / "cars_test_annos_withlabels.mat" + + create_image_folder( + root=base_folder, + name=images_folder_name, + file_name_fn=lambda image_index: f"{image_index:5d}.jpg", + num_examples=num_examples, + ) + + classes = np.random.randint(1, num_classes + 1, num_examples, dtype=np.uint8) + fnames = [f"{i:5d}.jpg" for i in range(num_examples)] + rec_array = fromarrays( + [classes, fnames], + names=["class", "fname"], + ) + io.savemat(annotations_mat_path, {"annotations": rec_array}) + + random_class_names = ["random_name"] * num_classes + io.savemat(devkit / "cars_meta.mat", {"class_names": random_class_names}) + + return num_examples \ No newline at end of file diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index c3787c7b50e..86da735adc8 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -11,7 +11,7 @@ class _StanfordCarsLabelReader(IterDataPipe[Tuple[np.ndarray, int]]): def __init__(self, datapipe: IterDataPipe[Dict[str, Any]]) -> None: self.datapipe = datapipe - def __iter__(self) -> Iterator[Tuple[str, Dict[str, str]]]: + def __iter__(self) -> Iterator[Tuple[str]]: for _, file in self.datapipe: file = iter(read_mat(file, squeeze_me=True)["annotations"]) for line in file: @@ -56,7 +56,7 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: resources.append(HttpResource(url=self._URLS["car_devkit"], sha256=self._CHECKSUM["car_devkit"])) return resources - def _prepare_sample(self, data): + def _prepare_sample(self, data:Tuple(IterDataPipe)) -> Dict(str, Any): image, target = data image_path, image_buffer = image image = EncodedImage.from_file(image_buffer) From b0b217d7e46b06e9de75eeb392dee5e4c8306771 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 10:21:37 +0530 Subject: [PATCH 23/53] lint (end of line issue) --- .../prototype/datasets/_builtin/stanford-cars.categories | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford-cars.categories b/torchvision/prototype/datasets/_builtin/stanford-cars.categories index 54aa513e43c..e54040f7779 100644 --- a/torchvision/prototype/datasets/_builtin/stanford-cars.categories +++ b/torchvision/prototype/datasets/_builtin/stanford-cars.categories @@ -193,4 +193,4 @@ Volkswagen Beetle Hatchback 2012 Volvo C30 Hatchback 2012 Volvo 240 Sedan 1993 Volvo XC90 SUV 2007 -smart fortwo Convertible 2012 \ No newline at end of file +smart fortwo Convertible 2012 From a067e4bab3b5e88eb6fb7730da7817c4bd1dc4c3 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 10:23:38 +0530 Subject: [PATCH 24/53] lint (end of line issue) --- test/builtin_dataset_mocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index 21af3eba03f..1a2b01ab4cd 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1429,4 +1429,4 @@ def stanford_cars_test(info,root,config): random_class_names = ["random_name"] * num_classes io.savemat(devkit / "cars_meta.mat", {"class_names": random_class_names}) - return num_examples \ No newline at end of file + return num_examples From e792ed71251c1a9fcceeb70ca11e090efe8f1006 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 10:27:52 +0530 Subject: [PATCH 25/53] minor changes --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 86da735adc8..a7badb16fa0 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -56,7 +56,7 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: resources.append(HttpResource(url=self._URLS["car_devkit"], sha256=self._CHECKSUM["car_devkit"])) return resources - def _prepare_sample(self, data:Tuple(IterDataPipe)) -> Dict(str, Any): + def _prepare_sample(self, data: Tuple(IterDataPipe)) -> Dict(str, Any): image, target = data image_path, image_buffer = image image = EncodedImage.from_file(image_buffer) From 8d0eaa5c1911449b7a596d04820b589df802dab8 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 10:31:10 +0530 Subject: [PATCH 26/53] mock dataset formatting --- test/builtin_dataset_mocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index 1a2b01ab4cd..eb7e5932f5a 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1393,7 +1393,7 @@ def pcam(info, root, config): @register_mock -def stanford_cars_test(info,root,config): +def stanford_cars_test(info, root, config): import scipy.io as io from np.core.records import fromarrays From 4a6f1b15c872a77e0edefe6fa8f5acb4d0e90fed Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 10:40:42 +0530 Subject: [PATCH 27/53] minor changes --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index a7badb16fa0..29605911895 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -44,7 +44,7 @@ def _make_info(self) -> DatasetInfo: "car_devkit": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", } - def resources(self, config: DatasetConfig) -> List[OnlineResource]: + def resources(self, config: DatasetConfig) -> List[HttpResource]: resources = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])] if config.split == "test": resources.append( @@ -56,7 +56,7 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: resources.append(HttpResource(url=self._URLS["car_devkit"], sha256=self._CHECKSUM["car_devkit"])) return resources - def _prepare_sample(self, data: Tuple(IterDataPipe)) -> Dict(str, Any): + def _prepare_sample(self, data: Tuple[IterDataPipe]) -> Dict[str, Any]: image, target = data image_path, image_buffer = image image = EncodedImage.from_file(image_buffer) From c4cfad4b7e7fd18c8b93086d7e1db56b39620c5b Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 11:13:30 +0530 Subject: [PATCH 28/53] hopefully final --- test/builtin_dataset_mocks.py | 2 +- .../prototype/datasets/_builtin/stanford_cars.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index eb7e5932f5a..ef70898ec31 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1393,7 +1393,7 @@ def pcam(info, root, config): @register_mock -def stanford_cars_test(info, root, config): +def stanford_cars(info, root, config): import scipy.io as io from np.core.records import fromarrays diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 29605911895..6bc26c0384d 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -2,7 +2,7 @@ import numpy as np from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper -from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource +from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat from torchvision.prototype.features import BoundingBox, EncodedImage @@ -33,14 +33,14 @@ def _make_info(self) -> DatasetInfo: _URLS = { "train": f"{_URL_ROOT}car196/cars_train.tgz", "test": f"{_URL_ROOT}car196/cars_test.tgz", - "test_ground_truth": f"{_URL_ROOT}car196/cars_test_annos_withlabels.mat", - "devkit": f"{_URL_ROOT}cars/car_devkit.tgz", + "cars_test_annos_withlabels": f"{_URL_ROOT}car196/cars_test_annos_withlabels.mat", + "car_devkit": f"{_URL_ROOT}cars/car_devkit.tgz", } _CHECKSUM = { "train": "b97deb463af7d58b6bfaa18b2a4de9829f0f79e8ce663dfa9261bf7810e9accd", "test": "bffea656d6f425cba3c91c6d83336e4c5f86c6cffd8975b0f375d3a10da8e243", - "cars_test_annos_withlabels.mat": "790f75be8ea34eeded134cc559332baf23e30e91367e9ddca97d26ed9b895f05", + "cars_test_annos_withlabels": "790f75be8ea34eeded134cc559332baf23e30e91367e9ddca97d26ed9b895f05", "car_devkit": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", } @@ -54,11 +54,13 @@ def resources(self, config: DatasetConfig) -> List[HttpResource]: ) else: resources.append(HttpResource(url=self._URLS["car_devkit"], sha256=self._CHECKSUM["car_devkit"])) - return resources + + return resources def _prepare_sample(self, data: Tuple[IterDataPipe]) -> Dict[str, Any]: image, target = data image_path, image_buffer = image + print(type(image_path),type(image_buffer)) image = EncodedImage.from_file(image_buffer) index = image_path[-9:-4] index = int(image_path[-9:-4]) - 1 From 89fef0149581e86a626067a87aa0f3a4234fd794 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 12:06:53 +0530 Subject: [PATCH 29/53] formatting --- test/builtin_dataset_mocks.py | 2 +- torchvision/prototype/datasets/_builtin/stanford_cars.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index ef70898ec31..4bfbeeea12c 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1395,7 +1395,7 @@ def pcam(info, root, config): @register_mock def stanford_cars(info, root, config): import scipy.io as io - from np.core.records import fromarrays + from numpy.core.records import fromarrays num_examples = {"train": 5, "test": 7}[config["split"]] num_classes = 3 diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 6bc26c0384d..7dc69e6dce1 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -54,13 +54,13 @@ def resources(self, config: DatasetConfig) -> List[HttpResource]: ) else: resources.append(HttpResource(url=self._URLS["car_devkit"], sha256=self._CHECKSUM["car_devkit"])) - + return resources def _prepare_sample(self, data: Tuple[IterDataPipe]) -> Dict[str, Any]: image, target = data image_path, image_buffer = image - print(type(image_path),type(image_buffer)) + print(type(image_path), type(image_buffer)) image = EncodedImage.from_file(image_buffer) index = image_path[-9:-4] index = int(image_path[-9:-4]) - 1 From f3bfc999a0f45c66499662e01963acdb5f95f4de Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 13:32:55 +0530 Subject: [PATCH 30/53] prototype test --- test/builtin_dataset_mocks.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index 4bfbeeea12c..4da8d0617ab 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1418,15 +1418,17 @@ def stanford_cars(info, root, config): num_examples=num_examples, ) + make_tar(root=base_folder, name=f"cars_{config.split}.tgz", files_or_dirs=images_folder_name) + bbox = np.random.randint(1, 200, num_examples, dtype=np.uint8) classes = np.random.randint(1, num_classes + 1, num_examples, dtype=np.uint8) fnames = [f"{i:5d}.jpg" for i in range(num_examples)] rec_array = fromarrays( - [classes, fnames], - names=["class", "fname"], + [bbox, bbox, bbox, bbox, classes, fnames], + names=["bbox_x1", "bbox_y1", "bbox_x2", "bbox_y2", "class", "fname"], ) - io.savemat(annotations_mat_path, {"annotations": rec_array}) - random_class_names = ["random_name"] * num_classes - io.savemat(devkit / "cars_meta.mat", {"class_names": random_class_names}) + io.savemat(annotations_mat_path, {"annotations": rec_array}) + if config.split == "train": + make_tar(root=base_folder, files_or_dirs=annotations_mat_path.parent) return num_examples From a9ecaf9b47169345c37ed93f895d2bfd3fc32c99 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 13:37:33 +0530 Subject: [PATCH 31/53] prototype test --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 7dc69e6dce1..323a08d1631 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -57,10 +57,9 @@ def resources(self, config: DatasetConfig) -> List[HttpResource]: return resources - def _prepare_sample(self, data: Tuple[IterDataPipe]) -> Dict[str, Any]: + def _prepare_sample(self, data: Tuple[IterDataPipe,Tuple[int,int,int,int,int,str]]) -> Dict[str, Any]: image, target = data image_path, image_buffer = image - print(type(image_path), type(image_buffer)) image = EncodedImage.from_file(image_buffer) index = image_path[-9:-4] index = int(image_path[-9:-4]) - 1 @@ -68,7 +67,6 @@ def _prepare_sample(self, data: Tuple[IterDataPipe]) -> Dict[str, Any]: return dict( index=index, image_path=image_path, - index1=target[-1], image=image, classification_label=target[4] - 1, bounding_box=BoundingBox( From 3b98146b30c84340b0526bf0c724936da75fe704 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 13:58:14 +0530 Subject: [PATCH 32/53] prototype test --- test/builtin_dataset_mocks.py | 4 ++-- torchvision/prototype/datasets/_builtin/stanford_cars.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index 4da8d0617ab..88892214a2f 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1418,7 +1418,7 @@ def stanford_cars(info, root, config): num_examples=num_examples, ) - make_tar(root=base_folder, name=f"cars_{config.split}.tgz", files_or_dirs=images_folder_name) + make_tar(base_folder, f"cars_{config.split}.tgz", images_folder_name) bbox = np.random.randint(1, 200, num_examples, dtype=np.uint8) classes = np.random.randint(1, num_classes + 1, num_examples, dtype=np.uint8) fnames = [f"{i:5d}.jpg" for i in range(num_examples)] @@ -1429,6 +1429,6 @@ def stanford_cars(info, root, config): io.savemat(annotations_mat_path, {"annotations": rec_array}) if config.split == "train": - make_tar(root=base_folder, files_or_dirs=annotations_mat_path.parent) + make_tar(base_folder, "cars_devkit.tgz", annotations_mat_path.parent) return num_examples diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 323a08d1631..2cde2c9232a 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -57,7 +57,7 @@ def resources(self, config: DatasetConfig) -> List[HttpResource]: return resources - def _prepare_sample(self, data: Tuple[IterDataPipe,Tuple[int,int,int,int,int,str]]) -> Dict[str, Any]: + def _prepare_sample(self, data: Tuple[IterDataPipe, Tuple[Any]]) -> Dict[str, Any]: image, target = data image_path, image_buffer = image image = EncodedImage.from_file(image_buffer) From e39a8403919798af80ebb0de79f16c02ecc13fcc Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 14:25:07 +0530 Subject: [PATCH 33/53] prototype test --- test/builtin_dataset_mocks.py | 2 +- torchvision/prototype/datasets/_builtin/stanford_cars.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index 88892214a2f..4d6647af118 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1399,7 +1399,7 @@ def stanford_cars(info, root, config): num_examples = {"train": 5, "test": 7}[config["split"]] num_classes = 3 - base_folder = pathlib.Path(root) / "stanford-cars" + base_folder = pathlib.Path(root) devkit = base_folder / "devkit" devkit.mkdir(parents=True) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 2cde2c9232a..cdc90ab7873 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -57,7 +57,7 @@ def resources(self, config: DatasetConfig) -> List[HttpResource]: return resources - def _prepare_sample(self, data: Tuple[IterDataPipe, Tuple[Any]]) -> Dict[str, Any]: + def _prepare_sample(self, data: Tuple[IterDataPipe, Tuple[int, int, int, int, int, str]]) -> Dict[str, Any]: image, target = data image_path, image_buffer = image image = EncodedImage.from_file(image_buffer) From e318c5029702007b46de2c0db4d5026ff0be7a8f Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 14:36:47 +0530 Subject: [PATCH 34/53] prototype test --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index cdc90ab7873..2a4ffe40521 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -2,7 +2,7 @@ import numpy as np from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper -from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource +from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat from torchvision.prototype.features import BoundingBox, EncodedImage @@ -44,7 +44,7 @@ def _make_info(self) -> DatasetInfo: "car_devkit": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", } - def resources(self, config: DatasetConfig) -> List[HttpResource]: + def resources(self, config: DatasetConfig) -> List[OnlineResource]: resources = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])] if config.split == "test": resources.append( From 729ec1742cdf72911d885cce7dd56d93141cb7e6 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 15:06:24 +0530 Subject: [PATCH 35/53] prototype test --- test/builtin_dataset_mocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index 4d6647af118..c75c43b4143 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1429,6 +1429,6 @@ def stanford_cars(info, root, config): io.savemat(annotations_mat_path, {"annotations": rec_array}) if config.split == "train": - make_tar(base_folder, "cars_devkit.tgz", annotations_mat_path.parent) + make_tar(base_folder, "cars_devkit.tgz", annotations_mat_path) return num_examples From 562973f547b39aebd6bae865830cf911409b48ae Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 15:26:35 +0530 Subject: [PATCH 36/53] prototype test --- test/builtin_dataset_mocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index c75c43b4143..3a1bc3a5953 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1429,6 +1429,6 @@ def stanford_cars(info, root, config): io.savemat(annotations_mat_path, {"annotations": rec_array}) if config.split == "train": - make_tar(base_folder, "cars_devkit.tgz", annotations_mat_path) + make_tar(base_folder, "cars_devkit.tgz", devkit) return num_examples From bbe9fb1c982c280e5bee23ba750677806390729b Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Sat, 5 Mar 2022 15:40:26 +0530 Subject: [PATCH 37/53] prototype test --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 2a4ffe40521..cdc90ab7873 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -2,7 +2,7 @@ import numpy as np from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper -from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource +from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat from torchvision.prototype.features import BoundingBox, EncodedImage @@ -44,7 +44,7 @@ def _make_info(self) -> DatasetInfo: "car_devkit": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", } - def resources(self, config: DatasetConfig) -> List[OnlineResource]: + def resources(self, config: DatasetConfig) -> List[HttpResource]: resources = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])] if config.split == "test": resources.append( From 47d8403273e6bb45b88d907204d1b615277cebbc Mon Sep 17 00:00:00 2001 From: Abhijit Deo <72816663+abhi-glitchhg@users.noreply.github.com> Date: Sat, 5 Mar 2022 22:52:07 +0530 Subject: [PATCH 38/53] Update builtin_dataset_mocks.py --- test/builtin_dataset_mocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index 3a1bc3a5953..c0dc2bab454 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1429,6 +1429,6 @@ def stanford_cars(info, root, config): io.savemat(annotations_mat_path, {"annotations": rec_array}) if config.split == "train": - make_tar(base_folder, "cars_devkit.tgz", devkit) + make_tar(base_folder, "car_devkit.tgz", devkit) return num_examples From 2136ec570735920bc09e2e4f24c1659a34bcfd6e Mon Sep 17 00:00:00 2001 From: Abhijit Deo <72816663+abhi-glitchhg@users.noreply.github.com> Date: Sun, 6 Mar 2022 20:55:12 +0530 Subject: [PATCH 39/53] Update stanford_cars.py --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index cdc90ab7873..4c191c71a39 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -61,7 +61,6 @@ def _prepare_sample(self, data: Tuple[IterDataPipe, Tuple[int, int, int, int, in image, target = data image_path, image_buffer = image image = EncodedImage.from_file(image_buffer) - index = image_path[-9:-4] index = int(image_path[-9:-4]) - 1 return dict( From be01e62973392c4cfe0253a64cea7667492b44e6 Mon Sep 17 00:00:00 2001 From: Abhijit Deo <72816663+abhi-glitchhg@users.noreply.github.com> Date: Mon, 7 Mar 2022 13:18:42 +0530 Subject: [PATCH 40/53] Update torchvision/prototype/datasets/_builtin/stanford_cars.py Co-authored-by: Philip Meier --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 4c191c71a39..5ed0236675d 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -11,11 +11,11 @@ class _StanfordCarsLabelReader(IterDataPipe[Tuple[np.ndarray, int]]): def __init__(self, datapipe: IterDataPipe[Dict[str, Any]]) -> None: self.datapipe = datapipe - def __iter__(self) -> Iterator[Tuple[str]]: + def __iter__(self) -> Iterator[Tuple[int, int, int, int, int, str]]: for _, file in self.datapipe: - file = iter(read_mat(file, squeeze_me=True)["annotations"]) - for line in file: - yield line + data = read_mat(file, squeeze_me=True) + for ann in data["annotations"]: + yield tuple(ann) class StanfordCars(Dataset): From 038c574c64df7bfaf927db92c9f7e9863cfbd11b Mon Sep 17 00:00:00 2001 From: Abhijit Deo <72816663+abhi-glitchhg@users.noreply.github.com> Date: Mon, 7 Mar 2022 13:18:52 +0530 Subject: [PATCH 41/53] Update torchvision/prototype/datasets/_builtin/stanford_cars.py Co-authored-by: Philip Meier --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 5ed0236675d..490a8212335 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -7,7 +7,7 @@ from torchvision.prototype.features import BoundingBox, EncodedImage -class _StanfordCarsLabelReader(IterDataPipe[Tuple[np.ndarray, int]]): +class StanfordCarsLabelReader(IterDataPipe[Tuple[int, int, int, int, int, str]]): def __init__(self, datapipe: IterDataPipe[Dict[str, Any]]) -> None: self.datapipe = datapipe From 54246bbe9ba5ac56af1ade810f108957fcd82898 Mon Sep 17 00:00:00 2001 From: Abhijit Deo <72816663+abhi-glitchhg@users.noreply.github.com> Date: Mon, 7 Mar 2022 13:19:03 +0530 Subject: [PATCH 42/53] Update torchvision/prototype/datasets/_builtin/stanford_cars.py Co-authored-by: Philip Meier --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 490a8212335..03bd31646d0 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -44,8 +44,8 @@ def _make_info(self) -> DatasetInfo: "car_devkit": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288", } - def resources(self, config: DatasetConfig) -> List[HttpResource]: - resources = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])] + def resources(self, config: DatasetConfig) -> List[OnlineResource]: + resources: List[OnlineResource] = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])] if config.split == "test": resources.append( HttpResource( From e5ffb52fd5a51ec6efac4cec99315f57606f8b19 Mon Sep 17 00:00:00 2001 From: Abhijit Deo <72816663+abhi-glitchhg@users.noreply.github.com> Date: Mon, 7 Mar 2022 13:19:53 +0530 Subject: [PATCH 43/53] Update torchvision/prototype/datasets/_builtin/stanford_cars.py Co-authored-by: Philip Meier --- .../datasets/_builtin/stanford_cars.py | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 03bd31646d0..7f1998772d1 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -57,27 +57,16 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: return resources - def _prepare_sample(self, data: Tuple[IterDataPipe, Tuple[int, int, int, int, int, str]]) -> Dict[str, Any]: + def _prepare_sample(self, data: Tuple[Tuple[str, BinaryIO], Tuple[int, int, int, int, int, str]]) -> Dict[str, Any]: image, target = data - image_path, image_buffer = image - image = EncodedImage.from_file(image_buffer) - index = int(image_path[-9:-4]) - 1 + path, buffer = image + image = EncodedImage.from_file(buffer) return dict( - index=index, - image_path=image_path, + path=path, image=image, - classification_label=target[4] - 1, - bounding_box=BoundingBox( - [ - target[0], - target[1], - target[2], - target[3], - ], - format="xyxy", - image_size=image.image_size, - ), + label=Label(target[4] - 1, categories=self.categories), + bounding_box=BoundingBox(target[:4], format="xyxy", image_size=image.image_size), ) def _make_datapipe( From 9ea352e59e996a4c051a6d877fba49abfd5bcfe1 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Mon, 7 Mar 2022 13:35:45 +0530 Subject: [PATCH 44/53] formatting and cleaning --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 7f1998772d1..cc776ab9754 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,10 +1,10 @@ -from typing import Any, Dict, List, Tuple, Iterator +from typing import Any, Dict, List, Tuple, Iterator, BinaryIO import numpy as np from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper -from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource +from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat -from torchvision.prototype.features import BoundingBox, EncodedImage +from torchvision.prototype.features import BoundingBox, EncodedImage, Label class StanfordCarsLabelReader(IterDataPipe[Tuple[int, int, int, int, int, str]]): @@ -79,7 +79,7 @@ def _make_datapipe( images_dp, targets_dp = resource_dps if config.split == "train": targets_dp = Filter(targets_dp, path_comparator("name", "cars_train_annos.mat")) - targets_dp = _StanfordCarsLabelReader(targets_dp) + targets_dp = StanfordCarsLabelReader(targets_dp) dp = Zipper(images_dp, targets_dp) dp = hint_sharding(dp) dp = hint_shuffling(dp) From 369c0210de9d5eb8160e4a1d6fa3a664b58fd656 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Mon, 7 Mar 2022 13:42:06 +0530 Subject: [PATCH 45/53] formatting and cleaning --- test/builtin_dataset_mocks.py | 11 +++++------ .../prototype/datasets/_builtin/stanford_cars.py | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index c0dc2bab454..d037f3f168f 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1399,9 +1399,8 @@ def stanford_cars(info, root, config): num_examples = {"train": 5, "test": 7}[config["split"]] num_classes = 3 - base_folder = pathlib.Path(root) - devkit = base_folder / "devkit" + devkit = root / "devkit" devkit.mkdir(parents=True) if config["split"] == "train": @@ -1409,16 +1408,16 @@ def stanford_cars(info, root, config): annotations_mat_path = devkit / "cars_train_annos.mat" else: images_folder_name = "cars_test" - annotations_mat_path = base_folder / "cars_test_annos_withlabels.mat" + annotations_mat_path = root / "cars_test_annos_withlabels.mat" create_image_folder( - root=base_folder, + root=root, name=images_folder_name, file_name_fn=lambda image_index: f"{image_index:5d}.jpg", num_examples=num_examples, ) - make_tar(base_folder, f"cars_{config.split}.tgz", images_folder_name) + make_tar(root, f"cars_{config.split}.tgz", images_folder_name) bbox = np.random.randint(1, 200, num_examples, dtype=np.uint8) classes = np.random.randint(1, num_classes + 1, num_examples, dtype=np.uint8) fnames = [f"{i:5d}.jpg" for i in range(num_examples)] @@ -1429,6 +1428,6 @@ def stanford_cars(info, root, config): io.savemat(annotations_mat_path, {"annotations": rec_array}) if config.split == "train": - make_tar(base_folder, "car_devkit.tgz", devkit) + make_tar(root, "car_devkit.tgz", devkit) return num_examples diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index cc776ab9754..821401ef745 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,6 +1,5 @@ from typing import Any, Dict, List, Tuple, Iterator, BinaryIO -import numpy as np from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat From 3a57e8b105d29b9b52438112b23566ba596b44c3 Mon Sep 17 00:00:00 2001 From: Abhijit Deo <72816663+abhi-glitchhg@users.noreply.github.com> Date: Mon, 7 Mar 2022 14:08:12 +0530 Subject: [PATCH 46/53] Update torchvision/prototype/datasets/_builtin/stanford_cars.py Co-authored-by: Philip Meier --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 821401ef745..8b0da9843ec 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -14,7 +14,7 @@ def __iter__(self) -> Iterator[Tuple[int, int, int, int, int, str]]: for _, file in self.datapipe: data = read_mat(file, squeeze_me=True) for ann in data["annotations"]: - yield tuple(ann) + yield tuple(ann) # type: ignore[misc] class StanfordCars(Dataset): From ce732761f6e8be9a3b5e0a1e8b7ec6316647831d Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Tue, 8 Mar 2022 21:29:35 +0530 Subject: [PATCH 47/53] generate catefories script --- .../_builtin/stanford-cars.categories | 197 +----------------- .../datasets/_builtin/stanford_cars.py | 17 ++ 2 files changed, 18 insertions(+), 196 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford-cars.categories b/torchvision/prototype/datasets/_builtin/stanford-cars.categories index e54040f7779..dee7740c0b3 100644 --- a/torchvision/prototype/datasets/_builtin/stanford-cars.categories +++ b/torchvision/prototype/datasets/_builtin/stanford-cars.categories @@ -1,196 +1 @@ -AM General Hummer SUV 2000 -Acura RL Sedan 2012 -Acura TL Sedan 2012 -Acura TL Type-S 2008 -Acura TSX Sedan 2012 -Acura Integra Type R 2001 -Acura ZDX Hatchback 2012 -Aston Martin V8 Vantage Convertible 2012 -Aston Martin V8 Vantage Coupe 2012 -Aston Martin Virage Convertible 2012 -Aston Martin Virage Coupe 2012 -Audi RS 4 Convertible 2008 -Audi A5 Coupe 2012 -Audi TTS Coupe 2012 -Audi R8 Coupe 2012 -Audi V8 Sedan 1994 -Audi 100 Sedan 1994 -Audi 100 Wagon 1994 -Audi TT Hatchback 2011 -Audi S6 Sedan 2011 -Audi S5 Convertible 2012 -Audi S5 Coupe 2012 -Audi S4 Sedan 2012 -Audi S4 Sedan 2007 -Audi TT RS Coupe 2012 -BMW ActiveHybrid 5 Sedan 2012 -BMW 1 Series Convertible 2012 -BMW 1 Series Coupe 2012 -BMW 3 Series Sedan 2012 -BMW 3 Series Wagon 2012 -BMW 6 Series Convertible 2007 -BMW X5 SUV 2007 -BMW X6 SUV 2012 -BMW M3 Coupe 2012 -BMW M5 Sedan 2010 -BMW M6 Convertible 2010 -BMW X3 SUV 2012 -BMW Z4 Convertible 2012 -Bentley Continental Supersports Conv. Convertible 2012 -Bentley Arnage Sedan 2009 -Bentley Mulsanne Sedan 2011 -Bentley Continental GT Coupe 2012 -Bentley Continental GT Coupe 2007 -Bentley Continental Flying Spur Sedan 2007 -Bugatti Veyron 16.4 Convertible 2009 -Bugatti Veyron 16.4 Coupe 2009 -Buick Regal GS 2012 -Buick Rainier SUV 2007 -Buick Verano Sedan 2012 -Buick Enclave SUV 2012 -Cadillac CTS-V Sedan 2012 -Cadillac SRX SUV 2012 -Cadillac Escalade EXT Crew Cab 2007 -Chevrolet Silverado 1500 Hybrid Crew Cab 2012 -Chevrolet Corvette Convertible 2012 -Chevrolet Corvette ZR1 2012 -Chevrolet Corvette Ron Fellows Edition Z06 2007 -Chevrolet Traverse SUV 2012 -Chevrolet Camaro Convertible 2012 -Chevrolet HHR SS 2010 -Chevrolet Impala Sedan 2007 -Chevrolet Tahoe Hybrid SUV 2012 -Chevrolet Sonic Sedan 2012 -Chevrolet Express Cargo Van 2007 -Chevrolet Avalanche Crew Cab 2012 -Chevrolet Cobalt SS 2010 -Chevrolet Malibu Hybrid Sedan 2010 -Chevrolet TrailBlazer SS 2009 -Chevrolet Silverado 2500HD Regular Cab 2012 -Chevrolet Silverado 1500 Classic Extended Cab 2007 -Chevrolet Express Van 2007 -Chevrolet Monte Carlo Coupe 2007 -Chevrolet Malibu Sedan 2007 -Chevrolet Silverado 1500 Extended Cab 2012 -Chevrolet Silverado 1500 Regular Cab 2012 -Chrysler Aspen SUV 2009 -Chrysler Sebring Convertible 2010 -Chrysler Town and Country Minivan 2012 -Chrysler 300 SRT-8 2010 -Chrysler Crossfire Convertible 2008 -Chrysler PT Cruiser Convertible 2008 -Daewoo Nubira Wagon 2002 -Dodge Caliber Wagon 2012 -Dodge Caliber Wagon 2007 -Dodge Caravan Minivan 1997 -Dodge Ram Pickup 3500 Crew Cab 2010 -Dodge Ram Pickup 3500 Quad Cab 2009 -Dodge Sprinter Cargo Van 2009 -Dodge Journey SUV 2012 -Dodge Dakota Crew Cab 2010 -Dodge Dakota Club Cab 2007 -Dodge Magnum Wagon 2008 -Dodge Challenger SRT8 2011 -Dodge Durango SUV 2012 -Dodge Durango SUV 2007 -Dodge Charger Sedan 2012 -Dodge Charger SRT-8 2009 -Eagle Talon Hatchback 1998 -FIAT 500 Abarth 2012 -FIAT 500 Convertible 2012 -Ferrari FF Coupe 2012 -Ferrari California Convertible 2012 -Ferrari 458 Italia Convertible 2012 -Ferrari 458 Italia Coupe 2012 -Fisker Karma Sedan 2012 -Ford F-450 Super Duty Crew Cab 2012 -Ford Mustang Convertible 2007 -Ford Freestar Minivan 2007 -Ford Expedition EL SUV 2009 -Ford Edge SUV 2012 -Ford Ranger SuperCab 2011 -Ford GT Coupe 2006 -Ford F-150 Regular Cab 2012 -Ford F-150 Regular Cab 2007 -Ford Focus Sedan 2007 -Ford E-Series Wagon Van 2012 -Ford Fiesta Sedan 2012 -GMC Terrain SUV 2012 -GMC Savana Van 2012 -GMC Yukon Hybrid SUV 2012 -GMC Acadia SUV 2012 -GMC Canyon Extended Cab 2012 -Geo Metro Convertible 1993 -HUMMER H3T Crew Cab 2010 -HUMMER H2 SUT Crew Cab 2009 -Honda Odyssey Minivan 2012 -Honda Odyssey Minivan 2007 -Honda Accord Coupe 2012 -Honda Accord Sedan 2012 -Hyundai Veloster Hatchback 2012 -Hyundai Santa Fe SUV 2012 -Hyundai Tucson SUV 2012 -Hyundai Veracruz SUV 2012 -Hyundai Sonata Hybrid Sedan 2012 -Hyundai Elantra Sedan 2007 -Hyundai Accent Sedan 2012 -Hyundai Genesis Sedan 2012 -Hyundai Sonata Sedan 2012 -Hyundai Elantra Touring Hatchback 2012 -Hyundai Azera Sedan 2012 -Infiniti G Coupe IPL 2012 -Infiniti QX56 SUV 2011 -Isuzu Ascender SUV 2008 -Jaguar XK XKR 2012 -Jeep Patriot SUV 2012 -Jeep Wrangler SUV 2012 -Jeep Liberty SUV 2012 -Jeep Grand Cherokee SUV 2012 -Jeep Compass SUV 2012 -Lamborghini Reventon Coupe 2008 -Lamborghini Aventador Coupe 2012 -Lamborghini Gallardo LP 570-4 Superleggera 2012 -Lamborghini Diablo Coupe 2001 -Land Rover Range Rover SUV 2012 -Land Rover LR2 SUV 2012 -Lincoln Town Car Sedan 2011 -MINI Cooper Roadster Convertible 2012 -Maybach Landaulet Convertible 2012 -Mazda Tribute SUV 2011 -McLaren MP4-12C Coupe 2012 -Mercedes-Benz 300-Class Convertible 1993 -Mercedes-Benz C-Class Sedan 2012 -Mercedes-Benz SL-Class Coupe 2009 -Mercedes-Benz E-Class Sedan 2012 -Mercedes-Benz S-Class Sedan 2012 -Mercedes-Benz Sprinter Van 2012 -Mitsubishi Lancer Sedan 2012 -Nissan Leaf Hatchback 2012 -Nissan NV Passenger Van 2012 -Nissan Juke Hatchback 2012 -Nissan 240SX Coupe 1998 -Plymouth Neon Coupe 1999 -Porsche Panamera Sedan 2012 -Ram C/V Cargo Van Minivan 2012 -Rolls-Royce Phantom Drophead Coupe Convertible 2012 -Rolls-Royce Ghost Sedan 2012 -Rolls-Royce Phantom Sedan 2012 -Scion xD Hatchback 2012 -Spyker C8 Convertible 2009 -Spyker C8 Coupe 2009 -Suzuki Aerio Sedan 2007 -Suzuki Kizashi Sedan 2012 -Suzuki SX4 Hatchback 2012 -Suzuki SX4 Sedan 2012 -Tesla Model S Sedan 2012 -Toyota Sequoia SUV 2012 -Toyota Camry Sedan 2012 -Toyota Corolla Sedan 2012 -Toyota 4Runner SUV 2012 -Volkswagen Golf Hatchback 2012 -Volkswagen Golf Hatchback 1991 -Volkswagen Beetle Hatchback 2012 -Volvo C30 Hatchback 2012 -Volvo 240 Sedan 1993 -Volvo XC90 SUV 2007 -smart fortwo Convertible 2012 +AM General Hummer SUV 2000,Acura RL Sedan 2012,Acura TL Sedan 2012,Acura TL Type-S 2008,Acura TSX Sedan 2012,Acura Integra Type R 2001,Acura ZDX Hatchback 2012,Aston Martin V8 Vantage Convertible 2012,Aston Martin V8 Vantage Coupe 2012,Aston Martin Virage Convertible 2012,Aston Martin Virage Coupe 2012,Audi RS 4 Convertible 2008,Audi A5 Coupe 2012,Audi TTS Coupe 2012,Audi R8 Coupe 2012,Audi V8 Sedan 1994,Audi 100 Sedan 1994,Audi 100 Wagon 1994,Audi TT Hatchback 2011,Audi S6 Sedan 2011,Audi S5 Convertible 2012,Audi S5 Coupe 2012,Audi S4 Sedan 2012,Audi S4 Sedan 2007,Audi TT RS Coupe 2012,BMW ActiveHybrid 5 Sedan 2012,BMW 1 Series Convertible 2012,BMW 1 Series Coupe 2012,BMW 3 Series Sedan 2012,BMW 3 Series Wagon 2012,BMW 6 Series Convertible 2007,BMW X5 SUV 2007,BMW X6 SUV 2012,BMW M3 Coupe 2012,BMW M5 Sedan 2010,BMW M6 Convertible 2010,BMW X3 SUV 2012,BMW Z4 Convertible 2012,Bentley Continental Supersports Conv. Convertible 2012,Bentley Arnage Sedan 2009,Bentley Mulsanne Sedan 2011,Bentley Continental GT Coupe 2012,Bentley Continental GT Coupe 2007,Bentley Continental Flying Spur Sedan 2007,Bugatti Veyron 16.4 Convertible 2009,Bugatti Veyron 16.4 Coupe 2009,Buick Regal GS 2012,Buick Rainier SUV 2007,Buick Verano Sedan 2012,Buick Enclave SUV 2012,Cadillac CTS-V Sedan 2012,Cadillac SRX SUV 2012,Cadillac Escalade EXT Crew Cab 2007,Chevrolet Silverado 1500 Hybrid Crew Cab 2012,Chevrolet Corvette Convertible 2012,Chevrolet Corvette ZR1 2012,Chevrolet Corvette Ron Fellows Edition Z06 2007,Chevrolet Traverse SUV 2012,Chevrolet Camaro Convertible 2012,Chevrolet HHR SS 2010,Chevrolet Impala Sedan 2007,Chevrolet Tahoe Hybrid SUV 2012,Chevrolet Sonic Sedan 2012,Chevrolet Express Cargo Van 2007,Chevrolet Avalanche Crew Cab 2012,Chevrolet Cobalt SS 2010,Chevrolet Malibu Hybrid Sedan 2010,Chevrolet TrailBlazer SS 2009,Chevrolet Silverado 2500HD Regular Cab 2012,Chevrolet Silverado 1500 Classic Extended Cab 2007,Chevrolet Express Van 2007,Chevrolet Monte Carlo Coupe 2007,Chevrolet Malibu Sedan 2007,Chevrolet Silverado 1500 Extended Cab 2012,Chevrolet Silverado 1500 Regular Cab 2012,Chrysler Aspen SUV 2009,Chrysler Sebring Convertible 2010,Chrysler Town and Country Minivan 2012,Chrysler 300 SRT-8 2010,Chrysler Crossfire Convertible 2008,Chrysler PT Cruiser Convertible 2008,Daewoo Nubira Wagon 2002,Dodge Caliber Wagon 2012,Dodge Caliber Wagon 2007,Dodge Caravan Minivan 1997,Dodge Ram Pickup 3500 Crew Cab 2010,Dodge Ram Pickup 3500 Quad Cab 2009,Dodge Sprinter Cargo Van 2009,Dodge Journey SUV 2012,Dodge Dakota Crew Cab 2010,Dodge Dakota Club Cab 2007,Dodge Magnum Wagon 2008,Dodge Challenger SRT8 2011,Dodge Durango SUV 2012,Dodge Durango SUV 2007,Dodge Charger Sedan 2012,Dodge Charger SRT-8 2009,Eagle Talon Hatchback 1998,FIAT 500 Abarth 2012,FIAT 500 Convertible 2012,Ferrari FF Coupe 2012,Ferrari California Convertible 2012,Ferrari 458 Italia Convertible 2012,Ferrari 458 Italia Coupe 2012,Fisker Karma Sedan 2012,Ford F-450 Super Duty Crew Cab 2012,Ford Mustang Convertible 2007,Ford Freestar Minivan 2007,Ford Expedition EL SUV 2009,Ford Edge SUV 2012,Ford Ranger SuperCab 2011,Ford GT Coupe 2006,Ford F-150 Regular Cab 2012,Ford F-150 Regular Cab 2007,Ford Focus Sedan 2007,Ford E-Series Wagon Van 2012,Ford Fiesta Sedan 2012,GMC Terrain SUV 2012,GMC Savana Van 2012,GMC Yukon Hybrid SUV 2012,GMC Acadia SUV 2012,GMC Canyon Extended Cab 2012,Geo Metro Convertible 1993,HUMMER H3T Crew Cab 2010,HUMMER H2 SUT Crew Cab 2009,Honda Odyssey Minivan 2012,Honda Odyssey Minivan 2007,Honda Accord Coupe 2012,Honda Accord Sedan 2012,Hyundai Veloster Hatchback 2012,Hyundai Santa Fe SUV 2012,Hyundai Tucson SUV 2012,Hyundai Veracruz SUV 2012,Hyundai Sonata Hybrid Sedan 2012,Hyundai Elantra Sedan 2007,Hyundai Accent Sedan 2012,Hyundai Genesis Sedan 2012,Hyundai Sonata Sedan 2012,Hyundai Elantra Touring Hatchback 2012,Hyundai Azera Sedan 2012,Infiniti G Coupe IPL 2012,Infiniti QX56 SUV 2011,Isuzu Ascender SUV 2008,Jaguar XK XKR 2012,Jeep Patriot SUV 2012,Jeep Wrangler SUV 2012,Jeep Liberty SUV 2012,Jeep Grand Cherokee SUV 2012,Jeep Compass SUV 2012,Lamborghini Reventon Coupe 2008,Lamborghini Aventador Coupe 2012,Lamborghini Gallardo LP 570-4 Superleggera 2012,Lamborghini Diablo Coupe 2001,Land Rover Range Rover SUV 2012,Land Rover LR2 SUV 2012,Lincoln Town Car Sedan 2011,MINI Cooper Roadster Convertible 2012,Maybach Landaulet Convertible 2012,Mazda Tribute SUV 2011,McLaren MP4-12C Coupe 2012,Mercedes-Benz 300-Class Convertible 1993,Mercedes-Benz C-Class Sedan 2012,Mercedes-Benz SL-Class Coupe 2009,Mercedes-Benz E-Class Sedan 2012,Mercedes-Benz S-Class Sedan 2012,Mercedes-Benz Sprinter Van 2012,Mitsubishi Lancer Sedan 2012,Nissan Leaf Hatchback 2012,Nissan NV Passenger Van 2012,Nissan Juke Hatchback 2012,Nissan 240SX Coupe 1998,Plymouth Neon Coupe 1999,Porsche Panamera Sedan 2012,Ram C/V Cargo Van Minivan 2012,Rolls-Royce Phantom Drophead Coupe Convertible 2012,Rolls-Royce Ghost Sedan 2012,Rolls-Royce Phantom Sedan 2012,Scion xD Hatchback 2012,Spyker C8 Convertible 2009,Spyker C8 Coupe 2009,Suzuki Aerio Sedan 2007,Suzuki Kizashi Sedan 2012,Suzuki SX4 Hatchback 2012,Suzuki SX4 Sedan 2012,Tesla Model S Sedan 2012,Toyota Sequoia SUV 2012,Toyota Camry Sedan 2012,Toyota Corolla Sedan 2012,Toyota 4Runner SUV 2012,Volkswagen Golf Hatchback 2012,Volkswagen Golf Hatchback 1991,Volkswagen Beetle Hatchback 2012,Volvo C30 Hatchback 2012,Volvo 240 Sedan 1993,Volvo XC90 SUV 2007,smart fortwo Convertible 2012 diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 821401ef745..25f805fb7c5 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,5 +1,8 @@ +import pathlib +from curses import meta from typing import Any, Dict, List, Tuple, Iterator, BinaryIO +from numpy import squeeze from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat @@ -56,6 +59,11 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: return resources + def prepare_categories(self, dp: IterDataPipe) -> List[str]: + _, file = dp + classes = list(read_mat(file, squeeze_me=True)["class_names"]) + return classes + def _prepare_sample(self, data: Tuple[Tuple[str, BinaryIO], Tuple[int, int, int, int, int, str]]) -> Dict[str, Any]: image, target = data path, buffer = image @@ -83,3 +91,12 @@ def _make_datapipe( dp = hint_sharding(dp) dp = hint_shuffling(dp) return Mapper(dp, self._prepare_sample) + + def _generate_categories(self, root: pathlib.Path) -> List[str]: + config = self.info.make_config(split="train") + resources = self.resources(config) + + devkit_dp = resources[1].load(root) + meta_dp = Filter(devkit_dp, path_comparator("name", "cars_meta.mat")) + print(meta_dp) + return Mapper(meta_dp, self.prepare_categories) From 6248e03a09a55d2b109795440c89a7d44339da3a Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Tue, 8 Mar 2022 21:39:18 +0530 Subject: [PATCH 48/53] generate categories script added --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 662c68a5584..23ecfb80255 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -1,8 +1,6 @@ import pathlib -from curses import meta from typing import Any, Dict, List, Tuple, Iterator, BinaryIO -from numpy import squeeze from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat From 05f81a6cbe0c11187ff146b937f6811c214dddca Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Tue, 8 Mar 2022 22:37:16 +0530 Subject: [PATCH 49/53] sigh --- .../_builtin/stanford-cars.categories | 197 +++++++++++++++++- 1 file changed, 196 insertions(+), 1 deletion(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford-cars.categories b/torchvision/prototype/datasets/_builtin/stanford-cars.categories index dee7740c0b3..e54040f7779 100644 --- a/torchvision/prototype/datasets/_builtin/stanford-cars.categories +++ b/torchvision/prototype/datasets/_builtin/stanford-cars.categories @@ -1 +1,196 @@ -AM General Hummer SUV 2000,Acura RL Sedan 2012,Acura TL Sedan 2012,Acura TL Type-S 2008,Acura TSX Sedan 2012,Acura Integra Type R 2001,Acura ZDX Hatchback 2012,Aston Martin V8 Vantage Convertible 2012,Aston Martin V8 Vantage Coupe 2012,Aston Martin Virage Convertible 2012,Aston Martin Virage Coupe 2012,Audi RS 4 Convertible 2008,Audi A5 Coupe 2012,Audi TTS Coupe 2012,Audi R8 Coupe 2012,Audi V8 Sedan 1994,Audi 100 Sedan 1994,Audi 100 Wagon 1994,Audi TT Hatchback 2011,Audi S6 Sedan 2011,Audi S5 Convertible 2012,Audi S5 Coupe 2012,Audi S4 Sedan 2012,Audi S4 Sedan 2007,Audi TT RS Coupe 2012,BMW ActiveHybrid 5 Sedan 2012,BMW 1 Series Convertible 2012,BMW 1 Series Coupe 2012,BMW 3 Series Sedan 2012,BMW 3 Series Wagon 2012,BMW 6 Series Convertible 2007,BMW X5 SUV 2007,BMW X6 SUV 2012,BMW M3 Coupe 2012,BMW M5 Sedan 2010,BMW M6 Convertible 2010,BMW X3 SUV 2012,BMW Z4 Convertible 2012,Bentley Continental Supersports Conv. Convertible 2012,Bentley Arnage Sedan 2009,Bentley Mulsanne Sedan 2011,Bentley Continental GT Coupe 2012,Bentley Continental GT Coupe 2007,Bentley Continental Flying Spur Sedan 2007,Bugatti Veyron 16.4 Convertible 2009,Bugatti Veyron 16.4 Coupe 2009,Buick Regal GS 2012,Buick Rainier SUV 2007,Buick Verano Sedan 2012,Buick Enclave SUV 2012,Cadillac CTS-V Sedan 2012,Cadillac SRX SUV 2012,Cadillac Escalade EXT Crew Cab 2007,Chevrolet Silverado 1500 Hybrid Crew Cab 2012,Chevrolet Corvette Convertible 2012,Chevrolet Corvette ZR1 2012,Chevrolet Corvette Ron Fellows Edition Z06 2007,Chevrolet Traverse SUV 2012,Chevrolet Camaro Convertible 2012,Chevrolet HHR SS 2010,Chevrolet Impala Sedan 2007,Chevrolet Tahoe Hybrid SUV 2012,Chevrolet Sonic Sedan 2012,Chevrolet Express Cargo Van 2007,Chevrolet Avalanche Crew Cab 2012,Chevrolet Cobalt SS 2010,Chevrolet Malibu Hybrid Sedan 2010,Chevrolet TrailBlazer SS 2009,Chevrolet Silverado 2500HD Regular Cab 2012,Chevrolet Silverado 1500 Classic Extended Cab 2007,Chevrolet Express Van 2007,Chevrolet Monte Carlo Coupe 2007,Chevrolet Malibu Sedan 2007,Chevrolet Silverado 1500 Extended Cab 2012,Chevrolet Silverado 1500 Regular Cab 2012,Chrysler Aspen SUV 2009,Chrysler Sebring Convertible 2010,Chrysler Town and Country Minivan 2012,Chrysler 300 SRT-8 2010,Chrysler Crossfire Convertible 2008,Chrysler PT Cruiser Convertible 2008,Daewoo Nubira Wagon 2002,Dodge Caliber Wagon 2012,Dodge Caliber Wagon 2007,Dodge Caravan Minivan 1997,Dodge Ram Pickup 3500 Crew Cab 2010,Dodge Ram Pickup 3500 Quad Cab 2009,Dodge Sprinter Cargo Van 2009,Dodge Journey SUV 2012,Dodge Dakota Crew Cab 2010,Dodge Dakota Club Cab 2007,Dodge Magnum Wagon 2008,Dodge Challenger SRT8 2011,Dodge Durango SUV 2012,Dodge Durango SUV 2007,Dodge Charger Sedan 2012,Dodge Charger SRT-8 2009,Eagle Talon Hatchback 1998,FIAT 500 Abarth 2012,FIAT 500 Convertible 2012,Ferrari FF Coupe 2012,Ferrari California Convertible 2012,Ferrari 458 Italia Convertible 2012,Ferrari 458 Italia Coupe 2012,Fisker Karma Sedan 2012,Ford F-450 Super Duty Crew Cab 2012,Ford Mustang Convertible 2007,Ford Freestar Minivan 2007,Ford Expedition EL SUV 2009,Ford Edge SUV 2012,Ford Ranger SuperCab 2011,Ford GT Coupe 2006,Ford F-150 Regular Cab 2012,Ford F-150 Regular Cab 2007,Ford Focus Sedan 2007,Ford E-Series Wagon Van 2012,Ford Fiesta Sedan 2012,GMC Terrain SUV 2012,GMC Savana Van 2012,GMC Yukon Hybrid SUV 2012,GMC Acadia SUV 2012,GMC Canyon Extended Cab 2012,Geo Metro Convertible 1993,HUMMER H3T Crew Cab 2010,HUMMER H2 SUT Crew Cab 2009,Honda Odyssey Minivan 2012,Honda Odyssey Minivan 2007,Honda Accord Coupe 2012,Honda Accord Sedan 2012,Hyundai Veloster Hatchback 2012,Hyundai Santa Fe SUV 2012,Hyundai Tucson SUV 2012,Hyundai Veracruz SUV 2012,Hyundai Sonata Hybrid Sedan 2012,Hyundai Elantra Sedan 2007,Hyundai Accent Sedan 2012,Hyundai Genesis Sedan 2012,Hyundai Sonata Sedan 2012,Hyundai Elantra Touring Hatchback 2012,Hyundai Azera Sedan 2012,Infiniti G Coupe IPL 2012,Infiniti QX56 SUV 2011,Isuzu Ascender SUV 2008,Jaguar XK XKR 2012,Jeep Patriot SUV 2012,Jeep Wrangler SUV 2012,Jeep Liberty SUV 2012,Jeep Grand Cherokee SUV 2012,Jeep Compass SUV 2012,Lamborghini Reventon Coupe 2008,Lamborghini Aventador Coupe 2012,Lamborghini Gallardo LP 570-4 Superleggera 2012,Lamborghini Diablo Coupe 2001,Land Rover Range Rover SUV 2012,Land Rover LR2 SUV 2012,Lincoln Town Car Sedan 2011,MINI Cooper Roadster Convertible 2012,Maybach Landaulet Convertible 2012,Mazda Tribute SUV 2011,McLaren MP4-12C Coupe 2012,Mercedes-Benz 300-Class Convertible 1993,Mercedes-Benz C-Class Sedan 2012,Mercedes-Benz SL-Class Coupe 2009,Mercedes-Benz E-Class Sedan 2012,Mercedes-Benz S-Class Sedan 2012,Mercedes-Benz Sprinter Van 2012,Mitsubishi Lancer Sedan 2012,Nissan Leaf Hatchback 2012,Nissan NV Passenger Van 2012,Nissan Juke Hatchback 2012,Nissan 240SX Coupe 1998,Plymouth Neon Coupe 1999,Porsche Panamera Sedan 2012,Ram C/V Cargo Van Minivan 2012,Rolls-Royce Phantom Drophead Coupe Convertible 2012,Rolls-Royce Ghost Sedan 2012,Rolls-Royce Phantom Sedan 2012,Scion xD Hatchback 2012,Spyker C8 Convertible 2009,Spyker C8 Coupe 2009,Suzuki Aerio Sedan 2007,Suzuki Kizashi Sedan 2012,Suzuki SX4 Hatchback 2012,Suzuki SX4 Sedan 2012,Tesla Model S Sedan 2012,Toyota Sequoia SUV 2012,Toyota Camry Sedan 2012,Toyota Corolla Sedan 2012,Toyota 4Runner SUV 2012,Volkswagen Golf Hatchback 2012,Volkswagen Golf Hatchback 1991,Volkswagen Beetle Hatchback 2012,Volvo C30 Hatchback 2012,Volvo 240 Sedan 1993,Volvo XC90 SUV 2007,smart fortwo Convertible 2012 +AM General Hummer SUV 2000 +Acura RL Sedan 2012 +Acura TL Sedan 2012 +Acura TL Type-S 2008 +Acura TSX Sedan 2012 +Acura Integra Type R 2001 +Acura ZDX Hatchback 2012 +Aston Martin V8 Vantage Convertible 2012 +Aston Martin V8 Vantage Coupe 2012 +Aston Martin Virage Convertible 2012 +Aston Martin Virage Coupe 2012 +Audi RS 4 Convertible 2008 +Audi A5 Coupe 2012 +Audi TTS Coupe 2012 +Audi R8 Coupe 2012 +Audi V8 Sedan 1994 +Audi 100 Sedan 1994 +Audi 100 Wagon 1994 +Audi TT Hatchback 2011 +Audi S6 Sedan 2011 +Audi S5 Convertible 2012 +Audi S5 Coupe 2012 +Audi S4 Sedan 2012 +Audi S4 Sedan 2007 +Audi TT RS Coupe 2012 +BMW ActiveHybrid 5 Sedan 2012 +BMW 1 Series Convertible 2012 +BMW 1 Series Coupe 2012 +BMW 3 Series Sedan 2012 +BMW 3 Series Wagon 2012 +BMW 6 Series Convertible 2007 +BMW X5 SUV 2007 +BMW X6 SUV 2012 +BMW M3 Coupe 2012 +BMW M5 Sedan 2010 +BMW M6 Convertible 2010 +BMW X3 SUV 2012 +BMW Z4 Convertible 2012 +Bentley Continental Supersports Conv. Convertible 2012 +Bentley Arnage Sedan 2009 +Bentley Mulsanne Sedan 2011 +Bentley Continental GT Coupe 2012 +Bentley Continental GT Coupe 2007 +Bentley Continental Flying Spur Sedan 2007 +Bugatti Veyron 16.4 Convertible 2009 +Bugatti Veyron 16.4 Coupe 2009 +Buick Regal GS 2012 +Buick Rainier SUV 2007 +Buick Verano Sedan 2012 +Buick Enclave SUV 2012 +Cadillac CTS-V Sedan 2012 +Cadillac SRX SUV 2012 +Cadillac Escalade EXT Crew Cab 2007 +Chevrolet Silverado 1500 Hybrid Crew Cab 2012 +Chevrolet Corvette Convertible 2012 +Chevrolet Corvette ZR1 2012 +Chevrolet Corvette Ron Fellows Edition Z06 2007 +Chevrolet Traverse SUV 2012 +Chevrolet Camaro Convertible 2012 +Chevrolet HHR SS 2010 +Chevrolet Impala Sedan 2007 +Chevrolet Tahoe Hybrid SUV 2012 +Chevrolet Sonic Sedan 2012 +Chevrolet Express Cargo Van 2007 +Chevrolet Avalanche Crew Cab 2012 +Chevrolet Cobalt SS 2010 +Chevrolet Malibu Hybrid Sedan 2010 +Chevrolet TrailBlazer SS 2009 +Chevrolet Silverado 2500HD Regular Cab 2012 +Chevrolet Silverado 1500 Classic Extended Cab 2007 +Chevrolet Express Van 2007 +Chevrolet Monte Carlo Coupe 2007 +Chevrolet Malibu Sedan 2007 +Chevrolet Silverado 1500 Extended Cab 2012 +Chevrolet Silverado 1500 Regular Cab 2012 +Chrysler Aspen SUV 2009 +Chrysler Sebring Convertible 2010 +Chrysler Town and Country Minivan 2012 +Chrysler 300 SRT-8 2010 +Chrysler Crossfire Convertible 2008 +Chrysler PT Cruiser Convertible 2008 +Daewoo Nubira Wagon 2002 +Dodge Caliber Wagon 2012 +Dodge Caliber Wagon 2007 +Dodge Caravan Minivan 1997 +Dodge Ram Pickup 3500 Crew Cab 2010 +Dodge Ram Pickup 3500 Quad Cab 2009 +Dodge Sprinter Cargo Van 2009 +Dodge Journey SUV 2012 +Dodge Dakota Crew Cab 2010 +Dodge Dakota Club Cab 2007 +Dodge Magnum Wagon 2008 +Dodge Challenger SRT8 2011 +Dodge Durango SUV 2012 +Dodge Durango SUV 2007 +Dodge Charger Sedan 2012 +Dodge Charger SRT-8 2009 +Eagle Talon Hatchback 1998 +FIAT 500 Abarth 2012 +FIAT 500 Convertible 2012 +Ferrari FF Coupe 2012 +Ferrari California Convertible 2012 +Ferrari 458 Italia Convertible 2012 +Ferrari 458 Italia Coupe 2012 +Fisker Karma Sedan 2012 +Ford F-450 Super Duty Crew Cab 2012 +Ford Mustang Convertible 2007 +Ford Freestar Minivan 2007 +Ford Expedition EL SUV 2009 +Ford Edge SUV 2012 +Ford Ranger SuperCab 2011 +Ford GT Coupe 2006 +Ford F-150 Regular Cab 2012 +Ford F-150 Regular Cab 2007 +Ford Focus Sedan 2007 +Ford E-Series Wagon Van 2012 +Ford Fiesta Sedan 2012 +GMC Terrain SUV 2012 +GMC Savana Van 2012 +GMC Yukon Hybrid SUV 2012 +GMC Acadia SUV 2012 +GMC Canyon Extended Cab 2012 +Geo Metro Convertible 1993 +HUMMER H3T Crew Cab 2010 +HUMMER H2 SUT Crew Cab 2009 +Honda Odyssey Minivan 2012 +Honda Odyssey Minivan 2007 +Honda Accord Coupe 2012 +Honda Accord Sedan 2012 +Hyundai Veloster Hatchback 2012 +Hyundai Santa Fe SUV 2012 +Hyundai Tucson SUV 2012 +Hyundai Veracruz SUV 2012 +Hyundai Sonata Hybrid Sedan 2012 +Hyundai Elantra Sedan 2007 +Hyundai Accent Sedan 2012 +Hyundai Genesis Sedan 2012 +Hyundai Sonata Sedan 2012 +Hyundai Elantra Touring Hatchback 2012 +Hyundai Azera Sedan 2012 +Infiniti G Coupe IPL 2012 +Infiniti QX56 SUV 2011 +Isuzu Ascender SUV 2008 +Jaguar XK XKR 2012 +Jeep Patriot SUV 2012 +Jeep Wrangler SUV 2012 +Jeep Liberty SUV 2012 +Jeep Grand Cherokee SUV 2012 +Jeep Compass SUV 2012 +Lamborghini Reventon Coupe 2008 +Lamborghini Aventador Coupe 2012 +Lamborghini Gallardo LP 570-4 Superleggera 2012 +Lamborghini Diablo Coupe 2001 +Land Rover Range Rover SUV 2012 +Land Rover LR2 SUV 2012 +Lincoln Town Car Sedan 2011 +MINI Cooper Roadster Convertible 2012 +Maybach Landaulet Convertible 2012 +Mazda Tribute SUV 2011 +McLaren MP4-12C Coupe 2012 +Mercedes-Benz 300-Class Convertible 1993 +Mercedes-Benz C-Class Sedan 2012 +Mercedes-Benz SL-Class Coupe 2009 +Mercedes-Benz E-Class Sedan 2012 +Mercedes-Benz S-Class Sedan 2012 +Mercedes-Benz Sprinter Van 2012 +Mitsubishi Lancer Sedan 2012 +Nissan Leaf Hatchback 2012 +Nissan NV Passenger Van 2012 +Nissan Juke Hatchback 2012 +Nissan 240SX Coupe 1998 +Plymouth Neon Coupe 1999 +Porsche Panamera Sedan 2012 +Ram C/V Cargo Van Minivan 2012 +Rolls-Royce Phantom Drophead Coupe Convertible 2012 +Rolls-Royce Ghost Sedan 2012 +Rolls-Royce Phantom Sedan 2012 +Scion xD Hatchback 2012 +Spyker C8 Convertible 2009 +Spyker C8 Coupe 2009 +Suzuki Aerio Sedan 2007 +Suzuki Kizashi Sedan 2012 +Suzuki SX4 Hatchback 2012 +Suzuki SX4 Sedan 2012 +Tesla Model S Sedan 2012 +Toyota Sequoia SUV 2012 +Toyota Camry Sedan 2012 +Toyota Corolla Sedan 2012 +Toyota 4Runner SUV 2012 +Volkswagen Golf Hatchback 2012 +Volkswagen Golf Hatchback 1991 +Volkswagen Beetle Hatchback 2012 +Volvo C30 Hatchback 2012 +Volvo 240 Sedan 1993 +Volvo XC90 SUV 2007 +smart fortwo Convertible 2012 From f4938bba88de893dda26678f99c6cd4216014279 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Tue, 8 Mar 2022 22:37:41 +0530 Subject: [PATCH 50/53] sigh --- .../prototype/datasets/_builtin/stanford_cars.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index 23ecfb80255..e1ceaba1117 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -57,11 +57,6 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: return resources - def prepare_categories(self, dp: IterDataPipe) -> List[str]: - _, file = dp - classes = list(read_mat(file, squeeze_me=True)["class_names"]) - return classes - def _prepare_sample(self, data: Tuple[Tuple[str, BinaryIO], Tuple[int, int, int, int, int, str]]) -> Dict[str, Any]: image, target = data path, buffer = image @@ -96,5 +91,6 @@ def _generate_categories(self, root: pathlib.Path) -> List[str]: devkit_dp = resources[1].load(root) meta_dp = Filter(devkit_dp, path_comparator("name", "cars_meta.mat")) - print(meta_dp) - return Mapper(meta_dp, self.prepare_categories) + _, meta_file = next(iter(meta_dp)) + + return list(read_mat(meta_file, squeeze_me=True)["class_names"]) From a8d88d1565c4d4f80e02ee5e727f0b04950627bf Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Tue, 8 Mar 2022 22:38:38 +0530 Subject: [PATCH 51/53] sigh --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index e1ceaba1117..cf6bd1c3da6 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -93,4 +93,4 @@ def _generate_categories(self, root: pathlib.Path) -> List[str]: meta_dp = Filter(devkit_dp, path_comparator("name", "cars_meta.mat")) _, meta_file = next(iter(meta_dp)) - return list(read_mat(meta_file, squeeze_me=True)["class_names"]) + return list(read_mat(meta_file, squeeze_me=True)["class_names"]) From 7e83969926059242154f903bb397b6ee72bee859 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Wed, 9 Mar 2022 15:43:45 +0530 Subject: [PATCH 52/53] minor changes --- test/builtin_dataset_mocks.py | 16 ++++++++-------- .../prototype/datasets/_builtin/stanford_cars.py | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/test/builtin_dataset_mocks.py b/test/builtin_dataset_mocks.py index d037f3f168f..a78ea098718 100644 --- a/test/builtin_dataset_mocks.py +++ b/test/builtin_dataset_mocks.py @@ -1397,8 +1397,8 @@ def stanford_cars(info, root, config): import scipy.io as io from numpy.core.records import fromarrays - num_examples = {"train": 5, "test": 7}[config["split"]] - num_classes = 3 + num_samples = {"train": 5, "test": 7}[config["split"]] + num_categories = 3 devkit = root / "devkit" devkit.mkdir(parents=True) @@ -1414,13 +1414,13 @@ def stanford_cars(info, root, config): root=root, name=images_folder_name, file_name_fn=lambda image_index: f"{image_index:5d}.jpg", - num_examples=num_examples, + num_examples=num_samples, ) make_tar(root, f"cars_{config.split}.tgz", images_folder_name) - bbox = np.random.randint(1, 200, num_examples, dtype=np.uint8) - classes = np.random.randint(1, num_classes + 1, num_examples, dtype=np.uint8) - fnames = [f"{i:5d}.jpg" for i in range(num_examples)] + bbox = np.random.randint(1, 200, num_samples, dtype=np.uint8) + classes = np.random.randint(1, num_categories + 1, num_samples, dtype=np.uint8) + fnames = [f"{i:5d}.jpg" for i in range(num_samples)] rec_array = fromarrays( [bbox, bbox, bbox, bbox, classes, fnames], names=["bbox_x1", "bbox_y1", "bbox_x2", "bbox_y2", "class", "fname"], @@ -1428,6 +1428,6 @@ def stanford_cars(info, root, config): io.savemat(annotations_mat_path, {"annotations": rec_array}) if config.split == "train": - make_tar(root, "car_devkit.tgz", devkit) + make_tar(root, "car_devkit.tgz", devkit, compression="gz") - return num_examples + return num_samples diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index cf6bd1c3da6..efc8811f2a0 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -46,15 +46,15 @@ def _make_info(self) -> DatasetInfo: def resources(self, config: DatasetConfig) -> List[OnlineResource]: resources: List[OnlineResource] = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])] - if config.split == "test": + if config.split == "train": + resources.append(HttpResource(url=self._URLS["car_devkit"], sha256=self._CHECKSUM["car_devkit"])) + + else: resources.append( HttpResource( self._URLS["cars_test_annos_withlabels"], sha256=self._CHECKSUM["cars_test_annos_withlabels"] ) - ) - else: - resources.append(HttpResource(url=self._URLS["car_devkit"], sha256=self._CHECKSUM["car_devkit"])) - + ) return resources def _prepare_sample(self, data: Tuple[Tuple[str, BinaryIO], Tuple[int, int, int, int, int, str]]) -> Dict[str, Any]: From 0302eb61dd945478a816272a9f6f7733e80d7089 Mon Sep 17 00:00:00 2001 From: abhijit_linux Date: Wed, 9 Mar 2022 15:47:47 +0530 Subject: [PATCH 53/53] linting --- torchvision/prototype/datasets/_builtin/stanford_cars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/prototype/datasets/_builtin/stanford_cars.py b/torchvision/prototype/datasets/_builtin/stanford_cars.py index efc8811f2a0..d4c4dcfe02b 100644 --- a/torchvision/prototype/datasets/_builtin/stanford_cars.py +++ b/torchvision/prototype/datasets/_builtin/stanford_cars.py @@ -54,7 +54,7 @@ def resources(self, config: DatasetConfig) -> List[OnlineResource]: HttpResource( self._URLS["cars_test_annos_withlabels"], sha256=self._CHECKSUM["cars_test_annos_withlabels"] ) - ) + ) return resources def _prepare_sample(self, data: Tuple[Tuple[str, BinaryIO], Tuple[int, int, int, int, int, str]]) -> Dict[str, Any]: