Skip to content

Commit

Permalink
add docstring and change prefetch_factor when num_workers=0
Browse files Browse the repository at this point in the history
  • Loading branch information
Jackwaterveg committed Jun 20, 2022
1 parent 78c1640 commit 64e1b3d
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 4 deletions.
4 changes: 2 additions & 2 deletions python/paddle/fluid/dataloader/dataloader_iter.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,8 +169,8 @@ def __init__(self, loader):
# NOTE: len(self._places) batch data compose as an output
# iteration, set blocking_queue can cache "self._prefetch_factor" iteration datas
# at most here
self._blocking_queue_capacity = (
self._prefetch_factor - 1) * len(self._places)
self._blocking_queue_capacity = self._prefetch_factor * len(
self._places)

self._init_thread()
self._shutdown = False
Expand Down
6 changes: 4 additions & 2 deletions python/paddle/fluid/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -351,10 +351,12 @@ class DataLoader(object):
num_workers(int): the number of subprocess to load data, 0 for no
subprocess used and loading data in main process. Default 0
use_buffer_reader (bool): whether to use bufferred reader.
If use_buffer_reader=True, the DataLoader would prefetch next
If use_buffer_reader=True, the DataLoader would prefetch
batch data asynchronously, so it would speed up data feeding
and occupies a little more CPU or GPU memory, i.e., the memory
of one batch input data. Default True.
prefetch_factor (int): Number of batch data the DataLoader would prefetch
if use_buffer_reader=True. Default 2.
use_shared_memory (bool): whether to use shared memory to speed up
putting data into inter-process queue, set :attr:`use_shared_memory`
as True only when the shared memory space on your machine(e.g.
Expand Down Expand Up @@ -485,7 +487,7 @@ def __init__(self,
num_workers = 0
self.num_workers = num_workers

assert prefetch_factor >= 2, "prefetch_factor should be greater than or equal to 2"
assert prefetch_factor > 0, "prefetch_factor should be a positive value"

self.use_shared_memory = use_shared_memory
if use_shared_memory and num_workers == 0:
Expand Down

0 comments on commit 64e1b3d

Please sign in to comment.