From 7bfe5bc006c7b4b70aa584494fa2a6175f6c9439 Mon Sep 17 00:00:00 2001 From: Siming Dai <908660116@qq.com> Date: Fri, 22 Nov 2024 14:58:10 +0800 Subject: [PATCH] Fix eval for map dataset (#9472) --- paddlenlp/trainer/trainer.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/paddlenlp/trainer/trainer.py b/paddlenlp/trainer/trainer.py index 372273b9e35e3..827f5c1807863 100644 --- a/paddlenlp/trainer/trainer.py +++ b/paddlenlp/trainer/trainer.py @@ -1593,20 +1593,13 @@ def _get_eval_sampler(self, eval_dataset: Dataset): drop_last=False, ) else: - drop_last = False - if self.args.pipeline_parallel_degree > 1: - drop_last = True - logger.warning( - "In parallel mode, the batch_size is strictly checked. set DistributedBatchSampler drop_last=True." - ) - return DistributedBatchSampler( eval_dataset, num_replicas=self.args.dataset_world_size, rank=self.args.dataset_rank, batch_size=self.args.per_device_eval_batch_size, shuffle=False, - drop_last=drop_last, + drop_last=False, ) def get_eval_dataloader(self, eval_dataset: Optional[Dataset] = None) -> DataLoader: