Skip to content

Commit

Permalink
Forward fix MPRS request index cycle
Browse files Browse the repository at this point in the history
Summary:
Per title. The bug is introduced in D46408394, we should only advance request index when limit has not been reached

And fix an OSS lint Error

Reviewed By: NivekT

Differential Revision: D46520106

fbshipit-source-id: c17277626fb749c60ffd0c260a3a2e7420d47a49
  • Loading branch information
ejguan authored and facebook-github-bot committed Jun 7, 2023
1 parent 40dd648 commit 901b483
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 4 deletions.
3 changes: 2 additions & 1 deletion torchdata/dataloader2/communication/iter.py
Original file line number Diff line number Diff line change
Expand Up @@ -431,16 +431,17 @@ def __iter__(self):
disabled_pipe[res_idx] = True
cnt_disabled_pipes += 1
disabled = True
req_idx = next(req_idx_cycle)
else:
# Only request if buffer is empty and has not reached the limit
if len(self.res_buffers[res_idx]) == 0 and (
self._limit is None or self._request_cnt < self._limit
):
self.datapipes[req_idx].protocol.request_next()
req_idx = next(req_idx_cycle)
self._request_cnt += 1
total_req_cnt += 1
total_res_cnt += 1
req_idx = next(req_idx_cycle)
res_idx = next(res_idx_cycle)
if not disabled:
yield response.value
Expand Down
4 changes: 1 addition & 3 deletions torchdata/dataloader2/reading_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -632,9 +632,7 @@ def restore(self, datapipe, serialized_state: bytes) -> DataPipe:
if hasattr(rs, "restore") and callable(rs.restore):
datapipe = rs.restore(datapipe, state)
else:
warnings.warn(
f"{rs} doesn't support `restore` from state, initialize from scratch"
)
warnings.warn(f"{rs} doesn't support `restore` from state, initialize from scratch")
datapipe = rs.initialize(datapipe)
return datapipe

Expand Down

0 comments on commit 901b483

Please sign in to comment.