Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use epoch_size instead of hardcoded values #174

Merged
merged 3 commits into from
Sep 21, 2018
Merged

Use epoch_size instead of hardcoded values #174

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b2402ae
Use epoch_size instead of hardcoded values
JanuszL Sep 17, 2018
9007dd0
Add check iterators size equality for data container test
JanuszL Sep 20, 2018
3d653e2
Fix data container test
JanuszL Sep 20, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 19 additions & 18 deletions dali/test/python/test_data_containers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths):
self.input = ops.CaffeReader(path = data_paths[0])

def define_graph(self):
images, labels = self.input()
images, labels = self.input(name="Reader")
return self.base_define_graph(images, labels)

class Caffe2ReadPipeline(CommonPipeline):
Expand All @@ -53,16 +53,16 @@ def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths):
self.input = ops.Caffe2Reader(path = data_paths[0])

def define_graph(self):
images, labels = self.input()
images, labels = self.input(name="Reader")
return self.base_define_graph(images, labels)

class FileReadPipeline(CommonPipeline):
def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths):
super(FileReadPipeline, self).__init__(batch_size, num_threads, device_id)
self.input = ops.FileReader(file_root = data_paths[0], file_list = data_paths[1])
self.input = ops.FileReader(file_root = data_paths[0])

def define_graph(self):
images, labels = self.input()
images, labels = self.input(name="Reader")
return self.base_define_graph(images, labels)

class TFRecordPipeline(CommonPipeline):
Expand All @@ -77,31 +77,34 @@ def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths):
})

def define_graph(self):
inputs = self.input()
inputs = self.input(name="Reader")
images = inputs["image/encoded"]
labels = inputs["image/class/label"]
return self.base_define_graph(images, labels)

test_data = {
FileReadPipeline: [["/data/imagenet/", "/data/imagenet/train-jpeg_map.txt", 1281167],
["/data/imagenet/", "/data/imagenet/val-jpeg_map.txt", 50000]],
MXNetReaderPipeline: [["/data/imagenet/train-480-val-256-recordio/train.rec", "/data/imagenet/train-480-val-256-recordio/train.idx", 1281167],
["/data/imagenet/train-480-val-256-recordio/val.rec", "/data/imagenet/train-480-val-256-recordio/val.idx", 50000]],
CaffeReadPipeline: [["/data/imagenet/train-lmdb-256x256", 1281167],
["/data/imagenet/val-lmdb-256x256", 50000]],
Caffe2ReadPipeline: [["/data/imagenet/train-c2lmdb-480", 1281167],
["/data/imagenet/val-c2lmdb-256", 50000]],
TFRecordPipeline: [["/data/imagenet/train-val-tfrecord-480/train-*", "/data/imagenet/train-val-tfrecord-480.idx/train-*", 1281167]]
FileReadPipeline: [["/data/imagenet/train-jpeg"],
["/data/imagenet/val-jpeg"]]
MXNetReaderPipeline: [["/data/imagenet/train-480-val-256-recordio/train.rec", "/data/imagenet/train-480-val-256-recordio/train.idx"],
["/data/imagenet/train-480-val-256-recordio/val.rec", "/data/imagenet/train-480-val-256-recordio/val.idx"]],
CaffeReadPipeline: [["/data/imagenet/train-lmdb-256x256"],
["/data/imagenet/val-lmdb-256x256"]],
Caffe2ReadPipeline: [["/data/imagenet/train-c2lmdb-480"],
["/data/imagenet/val-c2lmdb-256"]],
TFRecordPipeline: [["/data/imagenet/train-val-tfrecord-480/train-*", "/data/imagenet/train-val-tfrecord-480.idx/train-*"]]
}

N = 4 # number of GPUs
N = 1 # number of GPUs
BATCH_SIZE = 2048 # batch size
LOG_INTERVAL = 200 // BATCH_SIZE + 1

for pipe_name in test_data.keys():
data_set_len = len(test_data[pipe_name])
for i, data_set in enumerate(test_data[pipe_name]):
iters = data_set[-1]
pipes = [pipe_name(batch_size=BATCH_SIZE, num_threads=4, device_id = n, num_gpus = N, data_paths = data_set) for n in range(N)]
[pipe.build() for pipe in pipes]

iters = pipes[0].epoch_size("Reader")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we here check whether all of the iterators report the same size?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Otherwise lgtm

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

iters_tmp = iters
iters = iters // BATCH_SIZE
if iters_tmp != iters * BATCH_SIZE:
Expand All @@ -112,8 +115,6 @@ def define_graph(self):
if iters_tmp != iters * N:
iters += 1

pipes = [pipe_name(batch_size=BATCH_SIZE, num_threads=4, device_id = n, num_gpus = N, data_paths = data_set) for n in range(N)]
[pipe.build() for pipe in pipes]
print ("RUN {0}/{1}: {2}".format(i + 1, data_set_len, pipe_name.__name__))
print (data_set)
for j in range(iters):
Expand Down
4 changes: 2 additions & 2 deletions docs/examples/pytorch/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,13 +239,13 @@ def main():
pipe.build()
test_run = pipe.run()
from nvidia.dali.plugin.pytorch import DALIClassificationIterator
train_loader = DALIClassificationIterator(pipe, size = int(1281167 / args.world_size) )
train_loader = DALIClassificationIterator(pipe, size = int(pipe.epoch_size("Reader") / args.world_size) )

pipe = HybridValPipe(batch_size=args.batch_size, num_threads=args.workers, device_id = args.local_rank, data_dir = valdir, crop = crop_size, size = val_size)
pipe.build()
test_run = pipe.run()
from nvidia.dali.plugin.pytorch import DALIClassificationIterator
val_loader = DALIClassificationIterator(pipe, size = int(50000 / args.world_size) )
val_loader = DALIClassificationIterator(pipe, size = int(pipe.epoch_size("Reader") / args.world_size) )

if args.evaluate:
validate(val_loader, model, criterion)
Expand Down