From b2402aeb6d73e94c1f6703495cdf248e324fc02d Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Mon, 17 Sep 2018 13:11:37 +0200 Subject: [PATCH 1/3] Use epoch_size instead of hardcoded values Signed-off-by: Janusz Lisiecki --- dali/test/python/test_data_containers.py | 37 ++++++++++++------------ docs/examples/pytorch/main.py | 4 +-- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/dali/test/python/test_data_containers.py b/dali/test/python/test_data_containers.py index c6ef427fa47..75f7a4470cd 100644 --- a/dali/test/python/test_data_containers.py +++ b/dali/test/python/test_data_containers.py @@ -44,7 +44,7 @@ def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths): self.input = ops.CaffeReader(path = data_paths[0]) def define_graph(self): - images, labels = self.input() + images, labels = self.input(name="Reader") return self.base_define_graph(images, labels) class Caffe2ReadPipeline(CommonPipeline): @@ -53,16 +53,16 @@ def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths): self.input = ops.Caffe2Reader(path = data_paths[0]) def define_graph(self): - images, labels = self.input() + images, labels = self.input(name="Reader") return self.base_define_graph(images, labels) class FileReadPipeline(CommonPipeline): def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths): super(FileReadPipeline, self).__init__(batch_size, num_threads, device_id) - self.input = ops.FileReader(file_root = data_paths[0], file_list = data_paths[1]) + self.input = ops.FileReader(file_root = data_paths[0]) def define_graph(self): - images, labels = self.input() + images, labels = self.input(name="Reader") return self.base_define_graph(images, labels) class TFRecordPipeline(CommonPipeline): @@ -77,31 +77,34 @@ def __init__(self, batch_size, num_threads, device_id, num_gpus, data_paths): }) def define_graph(self): - inputs = self.input() + inputs = self.input(name="Reader") images = inputs["image/encoded"] labels = inputs["image/class/label"] return self.base_define_graph(images, labels) test_data = { - FileReadPipeline: [["/data/imagenet/", "/data/imagenet/train-jpeg_map.txt", 1281167], - ["/data/imagenet/", "/data/imagenet/val-jpeg_map.txt", 50000]], - MXNetReaderPipeline: [["/data/imagenet/train-480-val-256-recordio/train.rec", "/data/imagenet/train-480-val-256-recordio/train.idx", 1281167], - ["/data/imagenet/train-480-val-256-recordio/val.rec", "/data/imagenet/train-480-val-256-recordio/val.idx", 50000]], - CaffeReadPipeline: [["/data/imagenet/train-lmdb-256x256", 1281167], - ["/data/imagenet/val-lmdb-256x256", 50000]], - Caffe2ReadPipeline: [["/data/imagenet/train-c2lmdb-480", 1281167], - ["/data/imagenet/val-c2lmdb-256", 50000]], - TFRecordPipeline: [["/data/imagenet/train-val-tfrecord-480/train-*", "/data/imagenet/train-val-tfrecord-480.idx/train-*", 1281167]] + FileReadPipeline: [["/data/imagenet/train-jpeg"], + ["/data/imagenet/val-jpeg"]] + MXNetReaderPipeline: [["/data/imagenet/train-480-val-256-recordio/train.rec", "/data/imagenet/train-480-val-256-recordio/train.idx"], + ["/data/imagenet/train-480-val-256-recordio/val.rec", "/data/imagenet/train-480-val-256-recordio/val.idx"]], + CaffeReadPipeline: [["/data/imagenet/train-lmdb-256x256"], + ["/data/imagenet/val-lmdb-256x256"]], + Caffe2ReadPipeline: [["/data/imagenet/train-c2lmdb-480"], + ["/data/imagenet/val-c2lmdb-256"]], + TFRecordPipeline: [["/data/imagenet/train-val-tfrecord-480/train-*", "/data/imagenet/train-val-tfrecord-480.idx/train-*"]] } -N = 4 # number of GPUs +N = 1 # number of GPUs BATCH_SIZE = 2048 # batch size LOG_INTERVAL = 200 // BATCH_SIZE + 1 for pipe_name in test_data.keys(): data_set_len = len(test_data[pipe_name]) for i, data_set in enumerate(test_data[pipe_name]): - iters = data_set[-1] + pipes = [pipe_name(batch_size=BATCH_SIZE, num_threads=4, device_id = n, num_gpus = N, data_paths = data_set) for n in range(N)] + [pipe.build() for pipe in pipes] + + iters = pipes[0].epoch_size("Reader") iters_tmp = iters iters = iters // BATCH_SIZE if iters_tmp != iters * BATCH_SIZE: @@ -112,8 +115,6 @@ def define_graph(self): if iters_tmp != iters * N: iters += 1 - pipes = [pipe_name(batch_size=BATCH_SIZE, num_threads=4, device_id = n, num_gpus = N, data_paths = data_set) for n in range(N)] - [pipe.build() for pipe in pipes] print ("RUN {0}/{1}: {2}".format(i + 1, data_set_len, pipe_name.__name__)) print (data_set) for j in range(iters): diff --git a/docs/examples/pytorch/main.py b/docs/examples/pytorch/main.py index 7552ac0705c..a8684456844 100644 --- a/docs/examples/pytorch/main.py +++ b/docs/examples/pytorch/main.py @@ -239,13 +239,13 @@ def main(): pipe.build() test_run = pipe.run() from nvidia.dali.plugin.pytorch import DALIClassificationIterator - train_loader = DALIClassificationIterator(pipe, size = int(1281167 / args.world_size) ) + train_loader = DALIClassificationIterator(pipe, size = int(pipe.epoch_size("Reader") / args.world_size) ) pipe = HybridValPipe(batch_size=args.batch_size, num_threads=args.workers, device_id = args.local_rank, data_dir = valdir, crop = crop_size, size = val_size) pipe.build() test_run = pipe.run() from nvidia.dali.plugin.pytorch import DALIClassificationIterator - val_loader = DALIClassificationIterator(pipe, size = int(50000 / args.world_size) ) + val_loader = DALIClassificationIterator(pipe, size = int(pipe.epoch_size("Reader") / args.world_size) ) if args.evaluate: validate(val_loader, model, criterion) From 9007dd085f5b36927a05ac0f469ce83752bf5de2 Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Thu, 20 Sep 2018 08:05:59 +0200 Subject: [PATCH 2/3] Add check iterators size equality for data container test Signed-off-by: Janusz Lisiecki --- dali/test/python/test_data_containers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dali/test/python/test_data_containers.py b/dali/test/python/test_data_containers.py index 75f7a4470cd..325ff2f4d2d 100644 --- a/dali/test/python/test_data_containers.py +++ b/dali/test/python/test_data_containers.py @@ -105,6 +105,7 @@ def define_graph(self): [pipe.build() for pipe in pipes] iters = pipes[0].epoch_size("Reader") + assert(all(pipe.epoch_size("Reader") == iters for pipe in pipes) iters_tmp = iters iters = iters // BATCH_SIZE if iters_tmp != iters * BATCH_SIZE: From 3d653e2f132eec7ac88de7d73ac7d617fe175daf Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Thu, 20 Sep 2018 19:48:12 +0200 Subject: [PATCH 3/3] Fix data container test Signed-off-by: Janusz Lisiecki --- dali/test/python/test_data_containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dali/test/python/test_data_containers.py b/dali/test/python/test_data_containers.py index 325ff2f4d2d..0e997b8dfec 100644 --- a/dali/test/python/test_data_containers.py +++ b/dali/test/python/test_data_containers.py @@ -84,7 +84,7 @@ def define_graph(self): test_data = { FileReadPipeline: [["/data/imagenet/train-jpeg"], - ["/data/imagenet/val-jpeg"]] + ["/data/imagenet/val-jpeg"]], MXNetReaderPipeline: [["/data/imagenet/train-480-val-256-recordio/train.rec", "/data/imagenet/train-480-val-256-recordio/train.idx"], ["/data/imagenet/train-480-val-256-recordio/val.rec", "/data/imagenet/train-480-val-256-recordio/val.idx"]], CaffeReadPipeline: [["/data/imagenet/train-lmdb-256x256"], @@ -105,7 +105,7 @@ def define_graph(self): [pipe.build() for pipe in pipes] iters = pipes[0].epoch_size("Reader") - assert(all(pipe.epoch_size("Reader") == iters for pipe in pipes) + assert(all(pipe.epoch_size("Reader") == iters for pipe in pipes)) iters_tmp = iters iters = iters // BATCH_SIZE if iters_tmp != iters * BATCH_SIZE: