[eval model] store world logs per task

parlai/scripts/eval_model.py

@@ -118,13 +118,27 @@ def _save_eval_stats(opt, report):
         f.write("\n")  # for jq
 
 
+def get_task_world_logs(task, world_logs, is_multitask=False):
+    if not is_multitask:
+        return world_logs
+    else:
+        base_outfile, extension = os.path.splitext(world_logs)
+        return f'{base_outfile}_{task}{extension}'
+
+
 def _eval_single_world(opt, agent, task):
     logging.info(f'Evaluating task {task} using datatype {opt.get("datatype")}.')
     # set up world logger
-    world_logger = WorldLogger(opt) if opt['world_logs'] else None
-
     task_opt = opt.copy()  # copy opt since we're editing the task
     task_opt['task'] = task
+    # add task suffix in case of multi-tasking
+    if opt['world_logs']:
+        task_opt['world_logs'] = get_task_world_logs(
+            task, task_opt['world_logs'], is_multitask=len(opt['task'].split(',')) > 1
+        )
+
+    world_logger = WorldLogger(task_opt) if task_opt['world_logs'] else None
+
     world = create_task(task_opt, agent)  # create worlds for tasks
 
     # set up logging
@@ -161,10 +175,10 @@ def _eval_single_world(opt, agent, task):
         world_logger.reset()  # add final acts to logs
         if is_distributed():
             rank = get_rank()
-            base_outfile, extension = os.path.splitext(opt['world_logs'])
+            base_outfile, extension = os.path.splitext(task_opt['world_logs'])
             outfile = base_outfile + f'_{rank}' + extension
         else:
-            outfile = opt['world_logs']
+            outfile = task_opt['world_logs']
         world_logger.write(outfile, world, file_format=opt['save_format'])
 
     report = aggregate_unnamed_reports(all_gather_list(world.report()))

tests/test_eval_model.py

@@ -8,6 +8,7 @@
 import pytest
 import unittest
 import parlai.utils.testing as testing_utils
+from parlai.scripts.eval_model import get_task_world_logs
 
 
 class TestEvalModel(unittest.TestCase):
@@ -227,6 +228,34 @@ def test_save_report(self):
                 json_lines = f.readlines()
             assert len(json_lines) == 100
 
+    def test_save_multiple_logs(self):
+        """
+        Test that we can save multiple world_logs from eval model on multiple tasks.
+        """
+        with testing_utils.tempdir() as tmpdir:
+            log_report = os.path.join(tmpdir, 'world_logs.jsonl')
+            save_report = os.path.join(tmpdir, 'report')
+            multitask = 'integration_tests,blended_skill_talk'
+            opt = dict(
+                task=multitask,
+                model='repeat_label',
+                datatype='valid',
+                batchsize=97,
+                num_examples=100,
+                display_examples=False,
+                world_logs=log_report,
+                report_filename=save_report,
+            )
+            valid, test = testing_utils.eval_model(opt)
+
+            for task in multitask.split(','):
+                task_log_report = get_task_world_logs(
+                    task, log_report, is_multitask=True
+                )
+                with PathManager.open(task_log_report) as f:
+                    json_lines = f.readlines()
+                assert len(json_lines) == 100
+
 
 if __name__ == '__main__':
     unittest.main()