remove auto generated examples (NVIDIA#7510)

* explicitly remove autogenerated examples for data parallel evaluation

Signed-off-by: arendu <adithyare@nvidia.com>

* mark autogenrated and remove it for test

Signed-off-by: arendu <adithyare@nvidia.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Signed-off-by: arendu <adithyare@nvidia.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Signed-off-by: Sasha Meister <sasha.meister.work@gmail.com>

Author: 2 people

nemo/collections/nlp/data/language_modeling/megatron/gpt_sft_dataset.py

@@ -171,8 +171,13 @@ def __getitem__(self, idx):
         # idx may < 0 because we pad_samples_to_global_batch_size, e.g. id = -1
         if idx < 0:
             idx = len(self) + idx
+            auto_gen_idx = True
+        else:
+            auto_gen_idx = False
         try:
             example = self.indexed_dataset[idx]
+            if auto_gen_idx:
+                example['__AUTOGENERATED__'] = True
         except Exception as e:
             logging.error(f"Error while loading example {idx} from dataset {self.file_path}")
             raise e

nemo/collections/nlp/models/language_modeling/megatron_gpt_sft_model.py

@@ -492,7 +492,6 @@ def inference_epoch_end(self, outputs, mode, data_cfg):
             )
 
             # Remove duplicate examples due to distributed sampler.
-            inp_label_set = set()
             deduplicated_outputs = {
                 'preds': [],
                 'labels': [],
@@ -505,14 +504,16 @@ def inference_epoch_end(self, outputs, mode, data_cfg):
                     for pred, label, input, metadata in zip(
                         batch['preds'], batch['labels'], batch['inputs'], batch['metadata']
                     ):
-                        key = input + label
                         total_size += 1
-                        if key not in inp_label_set:
-                            inp_label_set.add(key)
+                        if not metadata.get("__AUTOGENERATED__", False):
                             deduplicated_outputs['preds'].append(pred)
                             deduplicated_outputs['labels'].append(label)
                             deduplicated_outputs['inputs'].append(input)
                             deduplicated_outputs['metadata'].append(metadata)
+                        else:
+                            logging.info(
+                                f"skipping autogenerated example example {input} prediction {pred} label {label}"
+                            )
 
             # Compute metric score
             metric_name = self.val_metric_name if mode == 'validation' else self.test_metric_name