Skip to content

Commit a566134

Browse files
SvecDavidAdam Blažek
authored andcommitted
Utilize DownloadableDataset (#27)
* Rewrite mnist_dataset to use DownloadableDataset * Remove abstract methods from mnist_dataset
1 parent 30e13e5 commit a566134

File tree

1 file changed

+3
-14
lines changed

1 file changed

+3
-14
lines changed

mnist_convnet/mnist_dataset.py

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import logging
22
import gzip
33
import struct
4-
import urllib.request
54
import os
65
import os.path as path
76

@@ -15,12 +14,13 @@
1514
'test_labels': 't10k-labels-idx1-ubyte.gz'}
1615

1716

18-
class MNISTDataset(cx.BaseDataset):
17+
class MNISTDataset(cx.DownloadableDataset):
1918
""" MNIST dataset for hand-written digits recognition."""
2019

21-
def _configure_dataset(self, data_root=path.join('datasets', '.mnist-data'), batch_size:int=100, **kwargs) -> None:
20+
def _configure_dataset(self, data_root=path.join('mnist_convnet', '.mnist-data'), batch_size:int=100, **kwargs) -> None:
2221
self._batch_size = batch_size
2322
self._data_root = data_root
23+
self._download_urls = [path.join(DOWNLOAD_ROOT, filename) for filename in FILENAMES.values()]
2424
self._data = {}
2525
self._data_loaded = False
2626

@@ -52,14 +52,3 @@ def test_stream(self) -> cx.Stream:
5252
for i in range(0, len(self._data['test_labels']), self._batch_size):
5353
yield {'images': self._data['test_images'][i: i + self._batch_size],
5454
'labels': self._data['test_labels'][i: i + self._batch_size]}
55-
56-
def download(self) -> None:
57-
"""Download method may be invoked with `cxflow dataset download <path-to-config>`."""
58-
for part in FILENAMES.values():
59-
target = path.join(self._data_root, part)
60-
if path.exists(target):
61-
logging.info('\t%s already exists', target)
62-
else:
63-
os.makedirs(self._data_root, exist_ok=True)
64-
logging.info('\tdownloading %s', target)
65-
urllib.request.urlretrieve(DOWNLOAD_ROOT+part, target)

0 commit comments

Comments
 (0)