Skip to content

Commit c590303

Browse files
datumboxpmeierNicolasHug
authored and
facebook-github-bot
committed
[fbsync] Implementation of Stanford Cars prototype dataset (#5442)
Summary: * basic stanford cars structure added. * added class in the init. * how to decode the images * fix typo in init * works partially * formatting * formatting * formatting * formatting * formatting * minor changes * minor changes * issue resolved * both datapipes are in sync * checksum dict keys changed * formatting * "ufmt formatting" * "ufmt formatting" * formatting * minor change * minor change * test added * lint (end of line issue) * lint (end of line issue) * minor changes * mock dataset formatting * minor changes * hopefully final * formatting * prototype test * prototype test * prototype test * prototype test * prototype test * prototype test * prototype test * prototype test * Update builtin_dataset_mocks.py * Update stanford_cars.py * Update torchvision/prototype/datasets/_builtin/stanford_cars.py * Update torchvision/prototype/datasets/_builtin/stanford_cars.py * Update torchvision/prototype/datasets/_builtin/stanford_cars.py * Update torchvision/prototype/datasets/_builtin/stanford_cars.py * formatting and cleaning * formatting and cleaning * Update torchvision/prototype/datasets/_builtin/stanford_cars.py * generate catefories script * generate categories script added * sigh * sigh * sigh * minor changes * linting Reviewed By: vmoens Differential Revision: D34879003 fbshipit-source-id: d9ffca1e9c2f5758dc0f4571f4e915fe45cb4b68 Co-authored-by: Philip Meier <github.pmeier@posteo.de> Co-authored-by: Philip Meier <github.pmeier@posteo.de> Co-authored-by: Philip Meier <github.pmeier@posteo.de> Co-authored-by: Philip Meier <github.pmeier@posteo.de> Co-authored-by: Philip Meier <github.pmeier@posteo.de> Co-authored-by: Philip Meier <github.pmeier@posteo.de> Co-authored-by: Nicolas Hug <contact@nicolas-hug.com>
1 parent 7b5c672 commit c590303

File tree

4 files changed

+334
-0
lines changed

4 files changed

+334
-0
lines changed

‎test/builtin_dataset_mocks.py

+41
Original file line numberDiff line numberDiff line change
@@ -1390,3 +1390,44 @@ def pcam(info, root, config):
13901390
compressed_file.write(compressed_data)
13911391

13921392
return num_images
1393+
1394+
1395+
@register_mock
1396+
def stanford_cars(info, root, config):
1397+
import scipy.io as io
1398+
from numpy.core.records import fromarrays
1399+
1400+
num_samples = {"train": 5, "test": 7}[config["split"]]
1401+
num_categories = 3
1402+
1403+
devkit = root / "devkit"
1404+
devkit.mkdir(parents=True)
1405+
1406+
if config["split"] == "train":
1407+
images_folder_name = "cars_train"
1408+
annotations_mat_path = devkit / "cars_train_annos.mat"
1409+
else:
1410+
images_folder_name = "cars_test"
1411+
annotations_mat_path = root / "cars_test_annos_withlabels.mat"
1412+
1413+
create_image_folder(
1414+
root=root,
1415+
name=images_folder_name,
1416+
file_name_fn=lambda image_index: f"{image_index:5d}.jpg",
1417+
num_examples=num_samples,
1418+
)
1419+
1420+
make_tar(root, f"cars_{config.split}.tgz", images_folder_name)
1421+
bbox = np.random.randint(1, 200, num_samples, dtype=np.uint8)
1422+
classes = np.random.randint(1, num_categories + 1, num_samples, dtype=np.uint8)
1423+
fnames = [f"{i:5d}.jpg" for i in range(num_samples)]
1424+
rec_array = fromarrays(
1425+
[bbox, bbox, bbox, bbox, classes, fnames],
1426+
names=["bbox_x1", "bbox_y1", "bbox_x2", "bbox_y2", "class", "fname"],
1427+
)
1428+
1429+
io.savemat(annotations_mat_path, {"annotations": rec_array})
1430+
if config.split == "train":
1431+
make_tar(root, "car_devkit.tgz", devkit, compression="gz")
1432+
1433+
return num_samples

‎torchvision/prototype/datasets/_builtin/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,6 @@
1515
from .pcam import PCAM
1616
from .sbd import SBD
1717
from .semeion import SEMEION
18+
from .stanford_cars import StanfordCars
1819
from .svhn import SVHN
1920
from .voc import VOC

‎torchvision/prototype/datasets/_builtin/stanford-cars.categories

+196
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
AM General Hummer SUV 2000
2+
Acura RL Sedan 2012
3+
Acura TL Sedan 2012
4+
Acura TL Type-S 2008
5+
Acura TSX Sedan 2012
6+
Acura Integra Type R 2001
7+
Acura ZDX Hatchback 2012
8+
Aston Martin V8 Vantage Convertible 2012
9+
Aston Martin V8 Vantage Coupe 2012
10+
Aston Martin Virage Convertible 2012
11+
Aston Martin Virage Coupe 2012
12+
Audi RS 4 Convertible 2008
13+
Audi A5 Coupe 2012
14+
Audi TTS Coupe 2012
15+
Audi R8 Coupe 2012
16+
Audi V8 Sedan 1994
17+
Audi 100 Sedan 1994
18+
Audi 100 Wagon 1994
19+
Audi TT Hatchback 2011
20+
Audi S6 Sedan 2011
21+
Audi S5 Convertible 2012
22+
Audi S5 Coupe 2012
23+
Audi S4 Sedan 2012
24+
Audi S4 Sedan 2007
25+
Audi TT RS Coupe 2012
26+
BMW ActiveHybrid 5 Sedan 2012
27+
BMW 1 Series Convertible 2012
28+
BMW 1 Series Coupe 2012
29+
BMW 3 Series Sedan 2012
30+
BMW 3 Series Wagon 2012
31+
BMW 6 Series Convertible 2007
32+
BMW X5 SUV 2007
33+
BMW X6 SUV 2012
34+
BMW M3 Coupe 2012
35+
BMW M5 Sedan 2010
36+
BMW M6 Convertible 2010
37+
BMW X3 SUV 2012
38+
BMW Z4 Convertible 2012
39+
Bentley Continental Supersports Conv. Convertible 2012
40+
Bentley Arnage Sedan 2009
41+
Bentley Mulsanne Sedan 2011
42+
Bentley Continental GT Coupe 2012
43+
Bentley Continental GT Coupe 2007
44+
Bentley Continental Flying Spur Sedan 2007
45+
Bugatti Veyron 16.4 Convertible 2009
46+
Bugatti Veyron 16.4 Coupe 2009
47+
Buick Regal GS 2012
48+
Buick Rainier SUV 2007
49+
Buick Verano Sedan 2012
50+
Buick Enclave SUV 2012
51+
Cadillac CTS-V Sedan 2012
52+
Cadillac SRX SUV 2012
53+
Cadillac Escalade EXT Crew Cab 2007
54+
Chevrolet Silverado 1500 Hybrid Crew Cab 2012
55+
Chevrolet Corvette Convertible 2012
56+
Chevrolet Corvette ZR1 2012
57+
Chevrolet Corvette Ron Fellows Edition Z06 2007
58+
Chevrolet Traverse SUV 2012
59+
Chevrolet Camaro Convertible 2012
60+
Chevrolet HHR SS 2010
61+
Chevrolet Impala Sedan 2007
62+
Chevrolet Tahoe Hybrid SUV 2012
63+
Chevrolet Sonic Sedan 2012
64+
Chevrolet Express Cargo Van 2007
65+
Chevrolet Avalanche Crew Cab 2012
66+
Chevrolet Cobalt SS 2010
67+
Chevrolet Malibu Hybrid Sedan 2010
68+
Chevrolet TrailBlazer SS 2009
69+
Chevrolet Silverado 2500HD Regular Cab 2012
70+
Chevrolet Silverado 1500 Classic Extended Cab 2007
71+
Chevrolet Express Van 2007
72+
Chevrolet Monte Carlo Coupe 2007
73+
Chevrolet Malibu Sedan 2007
74+
Chevrolet Silverado 1500 Extended Cab 2012
75+
Chevrolet Silverado 1500 Regular Cab 2012
76+
Chrysler Aspen SUV 2009
77+
Chrysler Sebring Convertible 2010
78+
Chrysler Town and Country Minivan 2012
79+
Chrysler 300 SRT-8 2010
80+
Chrysler Crossfire Convertible 2008
81+
Chrysler PT Cruiser Convertible 2008
82+
Daewoo Nubira Wagon 2002
83+
Dodge Caliber Wagon 2012
84+
Dodge Caliber Wagon 2007
85+
Dodge Caravan Minivan 1997
86+
Dodge Ram Pickup 3500 Crew Cab 2010
87+
Dodge Ram Pickup 3500 Quad Cab 2009
88+
Dodge Sprinter Cargo Van 2009
89+
Dodge Journey SUV 2012
90+
Dodge Dakota Crew Cab 2010
91+
Dodge Dakota Club Cab 2007
92+
Dodge Magnum Wagon 2008
93+
Dodge Challenger SRT8 2011
94+
Dodge Durango SUV 2012
95+
Dodge Durango SUV 2007
96+
Dodge Charger Sedan 2012
97+
Dodge Charger SRT-8 2009
98+
Eagle Talon Hatchback 1998
99+
FIAT 500 Abarth 2012
100+
FIAT 500 Convertible 2012
101+
Ferrari FF Coupe 2012
102+
Ferrari California Convertible 2012
103+
Ferrari 458 Italia Convertible 2012
104+
Ferrari 458 Italia Coupe 2012
105+
Fisker Karma Sedan 2012
106+
Ford F-450 Super Duty Crew Cab 2012
107+
Ford Mustang Convertible 2007
108+
Ford Freestar Minivan 2007
109+
Ford Expedition EL SUV 2009
110+
Ford Edge SUV 2012
111+
Ford Ranger SuperCab 2011
112+
Ford GT Coupe 2006
113+
Ford F-150 Regular Cab 2012
114+
Ford F-150 Regular Cab 2007
115+
Ford Focus Sedan 2007
116+
Ford E-Series Wagon Van 2012
117+
Ford Fiesta Sedan 2012
118+
GMC Terrain SUV 2012
119+
GMC Savana Van 2012
120+
GMC Yukon Hybrid SUV 2012
121+
GMC Acadia SUV 2012
122+
GMC Canyon Extended Cab 2012
123+
Geo Metro Convertible 1993
124+
HUMMER H3T Crew Cab 2010
125+
HUMMER H2 SUT Crew Cab 2009
126+
Honda Odyssey Minivan 2012
127+
Honda Odyssey Minivan 2007
128+
Honda Accord Coupe 2012
129+
Honda Accord Sedan 2012
130+
Hyundai Veloster Hatchback 2012
131+
Hyundai Santa Fe SUV 2012
132+
Hyundai Tucson SUV 2012
133+
Hyundai Veracruz SUV 2012
134+
Hyundai Sonata Hybrid Sedan 2012
135+
Hyundai Elantra Sedan 2007
136+
Hyundai Accent Sedan 2012
137+
Hyundai Genesis Sedan 2012
138+
Hyundai Sonata Sedan 2012
139+
Hyundai Elantra Touring Hatchback 2012
140+
Hyundai Azera Sedan 2012
141+
Infiniti G Coupe IPL 2012
142+
Infiniti QX56 SUV 2011
143+
Isuzu Ascender SUV 2008
144+
Jaguar XK XKR 2012
145+
Jeep Patriot SUV 2012
146+
Jeep Wrangler SUV 2012
147+
Jeep Liberty SUV 2012
148+
Jeep Grand Cherokee SUV 2012
149+
Jeep Compass SUV 2012
150+
Lamborghini Reventon Coupe 2008
151+
Lamborghini Aventador Coupe 2012
152+
Lamborghini Gallardo LP 570-4 Superleggera 2012
153+
Lamborghini Diablo Coupe 2001
154+
Land Rover Range Rover SUV 2012
155+
Land Rover LR2 SUV 2012
156+
Lincoln Town Car Sedan 2011
157+
MINI Cooper Roadster Convertible 2012
158+
Maybach Landaulet Convertible 2012
159+
Mazda Tribute SUV 2011
160+
McLaren MP4-12C Coupe 2012
161+
Mercedes-Benz 300-Class Convertible 1993
162+
Mercedes-Benz C-Class Sedan 2012
163+
Mercedes-Benz SL-Class Coupe 2009
164+
Mercedes-Benz E-Class Sedan 2012
165+
Mercedes-Benz S-Class Sedan 2012
166+
Mercedes-Benz Sprinter Van 2012
167+
Mitsubishi Lancer Sedan 2012
168+
Nissan Leaf Hatchback 2012
169+
Nissan NV Passenger Van 2012
170+
Nissan Juke Hatchback 2012
171+
Nissan 240SX Coupe 1998
172+
Plymouth Neon Coupe 1999
173+
Porsche Panamera Sedan 2012
174+
Ram C/V Cargo Van Minivan 2012
175+
Rolls-Royce Phantom Drophead Coupe Convertible 2012
176+
Rolls-Royce Ghost Sedan 2012
177+
Rolls-Royce Phantom Sedan 2012
178+
Scion xD Hatchback 2012
179+
Spyker C8 Convertible 2009
180+
Spyker C8 Coupe 2009
181+
Suzuki Aerio Sedan 2007
182+
Suzuki Kizashi Sedan 2012
183+
Suzuki SX4 Hatchback 2012
184+
Suzuki SX4 Sedan 2012
185+
Tesla Model S Sedan 2012
186+
Toyota Sequoia SUV 2012
187+
Toyota Camry Sedan 2012
188+
Toyota Corolla Sedan 2012
189+
Toyota 4Runner SUV 2012
190+
Volkswagen Golf Hatchback 2012
191+
Volkswagen Golf Hatchback 1991
192+
Volkswagen Beetle Hatchback 2012
193+
Volvo C30 Hatchback 2012
194+
Volvo 240 Sedan 1993
195+
Volvo XC90 SUV 2007
196+
smart fortwo Convertible 2012

‎torchvision/prototype/datasets/_builtin/stanford_cars.py

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
import pathlib
2+
from typing import Any, Dict, List, Tuple, Iterator, BinaryIO
3+
4+
from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper, Zipper
5+
from torchvision.prototype.datasets.utils import Dataset, DatasetConfig, DatasetInfo, HttpResource, OnlineResource
6+
from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling, path_comparator, read_mat
7+
from torchvision.prototype.features import BoundingBox, EncodedImage, Label
8+
9+
10+
class StanfordCarsLabelReader(IterDataPipe[Tuple[int, int, int, int, int, str]]):
11+
def __init__(self, datapipe: IterDataPipe[Dict[str, Any]]) -> None:
12+
self.datapipe = datapipe
13+
14+
def __iter__(self) -> Iterator[Tuple[int, int, int, int, int, str]]:
15+
for _, file in self.datapipe:
16+
data = read_mat(file, squeeze_me=True)
17+
for ann in data["annotations"]:
18+
yield tuple(ann) # type: ignore[misc]
19+
20+
21+
class StanfordCars(Dataset):
22+
def _make_info(self) -> DatasetInfo:
23+
return DatasetInfo(
24+
name="stanford-cars",
25+
homepage="https://ai.stanford.edu/~jkrause/cars/car_dataset.html",
26+
dependencies=("scipy",),
27+
valid_options=dict(
28+
split=("test", "train"),
29+
),
30+
)
31+
32+
_URL_ROOT = "https://ai.stanford.edu/~jkrause/"
33+
_URLS = {
34+
"train": f"{_URL_ROOT}car196/cars_train.tgz",
35+
"test": f"{_URL_ROOT}car196/cars_test.tgz",
36+
"cars_test_annos_withlabels": f"{_URL_ROOT}car196/cars_test_annos_withlabels.mat",
37+
"car_devkit": f"{_URL_ROOT}cars/car_devkit.tgz",
38+
}
39+
40+
_CHECKSUM = {
41+
"train": "b97deb463af7d58b6bfaa18b2a4de9829f0f79e8ce663dfa9261bf7810e9accd",
42+
"test": "bffea656d6f425cba3c91c6d83336e4c5f86c6cffd8975b0f375d3a10da8e243",
43+
"cars_test_annos_withlabels": "790f75be8ea34eeded134cc559332baf23e30e91367e9ddca97d26ed9b895f05",
44+
"car_devkit": "512b227b30e2f0a8aab9e09485786ab4479582073a144998da74d64b801fd288",
45+
}
46+
47+
def resources(self, config: DatasetConfig) -> List[OnlineResource]:
48+
resources: List[OnlineResource] = [HttpResource(self._URLS[config.split], sha256=self._CHECKSUM[config.split])]
49+
if config.split == "train":
50+
resources.append(HttpResource(url=self._URLS["car_devkit"], sha256=self._CHECKSUM["car_devkit"]))
51+
52+
else:
53+
resources.append(
54+
HttpResource(
55+
self._URLS["cars_test_annos_withlabels"], sha256=self._CHECKSUM["cars_test_annos_withlabels"]
56+
)
57+
)
58+
return resources
59+
60+
def _prepare_sample(self, data: Tuple[Tuple[str, BinaryIO], Tuple[int, int, int, int, int, str]]) -> Dict[str, Any]:
61+
image, target = data
62+
path, buffer = image
63+
image = EncodedImage.from_file(buffer)
64+
65+
return dict(
66+
path=path,
67+
image=image,
68+
label=Label(target[4] - 1, categories=self.categories),
69+
bounding_box=BoundingBox(target[:4], format="xyxy", image_size=image.image_size),
70+
)
71+
72+
def _make_datapipe(
73+
self,
74+
resource_dps: List[IterDataPipe],
75+
*,
76+
config: DatasetConfig,
77+
) -> IterDataPipe[Dict[str, Any]]:
78+
79+
images_dp, targets_dp = resource_dps
80+
if config.split == "train":
81+
targets_dp = Filter(targets_dp, path_comparator("name", "cars_train_annos.mat"))
82+
targets_dp = StanfordCarsLabelReader(targets_dp)
83+
dp = Zipper(images_dp, targets_dp)
84+
dp = hint_sharding(dp)
85+
dp = hint_shuffling(dp)
86+
return Mapper(dp, self._prepare_sample)
87+
88+
def _generate_categories(self, root: pathlib.Path) -> List[str]:
89+
config = self.info.make_config(split="train")
90+
resources = self.resources(config)
91+
92+
devkit_dp = resources[1].load(root)
93+
meta_dp = Filter(devkit_dp, path_comparator("name", "cars_meta.mat"))
94+
_, meta_file = next(iter(meta_dp))
95+
96+
return list(read_mat(meta_file, squeeze_me=True)["class_names"])

0 commit comments

Comments
 (0)