From f8748da33e42ddf9d8d6b6723db7e625403f327f Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Tue, 30 Nov 2021 13:37:07 -0800 Subject: [PATCH 1/5] check dir function added. --- parlai/utils/conversations.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/parlai/utils/conversations.py b/parlai/utils/conversations.py index 2346c3dd825..ed728292cb2 100644 --- a/parlai/utils/conversations.py +++ b/parlai/utils/conversations.py @@ -276,6 +276,14 @@ def _get_path(datapath): fle, _ = os.path.splitext(datapath) return fle + '.jsonl' + @staticmethod + def _check_parent_dir_exits(datapath): + parent_dir = os.path.dirname(datapath) + if os.path.exists(parent_dir) and os.path.isdir(parent_dir): + return + logging.info(f'Parent directory ({parent_dir}) did not exist and was created.') + os.makedirs(parent_dir) + @classmethod def save_conversations( cls, @@ -294,6 +302,7 @@ def save_conversations( each of which is comprised of a list of act pairs (i.e. a list dictionaries returned from one parley) """ + cls._check_parent_dir_exits(datapath) to_save = cls._get_path(datapath) context_ids = context_ids.strip().split(',') From 06f202a414f3384b182ac389acdc10ec2c979b53 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Wed, 1 Dec 2021 15:04:04 -0800 Subject: [PATCH 2/5] path manager --- parlai/utils/conversations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parlai/utils/conversations.py b/parlai/utils/conversations.py index ed728292cb2..37458f06299 100644 --- a/parlai/utils/conversations.py +++ b/parlai/utils/conversations.py @@ -279,10 +279,10 @@ def _get_path(datapath): @staticmethod def _check_parent_dir_exits(datapath): parent_dir = os.path.dirname(datapath) - if os.path.exists(parent_dir) and os.path.isdir(parent_dir): + if PathManager.exists(parent_dir) and PathManager.isdir(parent_dir): return logging.info(f'Parent directory ({parent_dir}) did not exist and was created.') - os.makedirs(parent_dir) + PathManager.mkdirs(parent_dir) @classmethod def save_conversations( From 023882f5bedf1b1b1ffdcde51fb7c92693ffb3f6 Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Wed, 1 Dec 2021 15:25:51 -0800 Subject: [PATCH 3/5] passing unit tests --- parlai/utils/conversations.py | 6 +- self_chat_output.html | 126 ++++++++++++++++++++++++++++++++++ self_chat_output.jsonl | 1 + self_chat_output.metadata | 1 + 4 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 self_chat_output.html create mode 100644 self_chat_output.jsonl create mode 100644 self_chat_output.metadata diff --git a/parlai/utils/conversations.py b/parlai/utils/conversations.py index 37458f06299..54c9142d8c4 100644 --- a/parlai/utils/conversations.py +++ b/parlai/utils/conversations.py @@ -279,7 +279,11 @@ def _get_path(datapath): @staticmethod def _check_parent_dir_exits(datapath): parent_dir = os.path.dirname(datapath) - if PathManager.exists(parent_dir) and PathManager.isdir(parent_dir): + if ( + not parent_dir + or PathManager.exists(parent_dir) + and PathManager.isdir(parent_dir) + ): return logging.info(f'Parent directory ({parent_dir}) did not exist and was created.') PathManager.mkdirs(parent_dir) diff --git a/self_chat_output.html b/self_chat_output.html new file mode 100644 index 00000000000..67dca77d083 --- /dev/null +++ b/self_chat_output.html @@ -0,0 +1,126 @@ + + + + Rendered HTML + + + +
    + +
  • +
    + +
    +
    +

    Hey there

    +
    +
    +
    • + +
  • +
    + +
    +
    +

    Hey there

    +
    +
    +
    • + +
  • +
    + +
    +
    +

    Hey there

    +
    +
    +
    • + +
  • +
    + +
    +
    +

    Hey there

    +
    +
    +
    • + +
  • +
    + +
    +
    +

    Hey there

    +
    +
    +
    • + +
    • +
+ + + \ No newline at end of file diff --git a/self_chat_output.jsonl b/self_chat_output.jsonl new file mode 100644 index 00000000000..792d56320e6 --- /dev/null +++ b/self_chat_output.jsonl @@ -0,0 +1 @@ +{"dialog": [[{"id": "FixedResponseAgent_1", "text": "Hey there", "episode_done": false}, {"id": "FixedResponseAgent_2", "text": "Hey there", "episode_done": false}], [{"id": "FixedResponseAgent_1", "text": "Hey there", "episode_done": false}, {"id": "FixedResponseAgent_2", "text": "Hey there", "episode_done": false}], [{"id": "FixedResponseAgent_1", "text": "Hey there", "episode_done": false}, {"id": "FixedResponseAgent_2", "text": "Hey there", "episode_done": false}], [{"id": "FixedResponseAgent_1", "text": "Hey there", "episode_done": false}, {"id": "FixedResponseAgent_2", "text": "Hey there", "episode_done": false}], [{"id": "FixedResponseAgent_1", "text": "Hey there", "episode_done": false}, {"id": "FixedResponseAgent_2", "text": "Hey there", "episode_done": false}]], "context": [{"text": "Hi!", "episode_done": false, "id": "context"}, {"text": "", "episode_done": false, "id": "context"}], "metadata_path": "self_chat_output.metadata"} diff --git a/self_chat_output.metadata b/self_chat_output.metadata new file mode 100644 index 00000000000..e6f02a1ad6d --- /dev/null +++ b/self_chat_output.metadata @@ -0,0 +1 @@ +{"date": "2021-12-01 15:25:29.530209", "opt": {"init_opt": null, "allow_missing_init_opts": false, "task": "self_chat", "download_path": null, "loglevel": "info", "datatype": "train", "image_mode": "raw", "hide_labels": false, "multitask_weights": [1], "batchsize": 1, "dynamic_batching": null, "verbose": false, "is_debug": false, "datapath": "/private/home/komeili/dev/ParlAI/data", "model": "fixed_response", "model_file": null, "init_model": null, "dict_class": null, "seed": 42, "display_examples": true, "display_add_fields": "", "selfchat_task": true, "num_self_chats": 1, "selfchat_max_turns": 6, "seed_messages_from_task": false, "seed_messages_from_file": null, "outfile": "self_chat_output", "save_format": "conversations", "partner_model_file": null, "partner_opt_file": null, "log_keep_fields": "all", "image_size": 256, "image_cropsize": 224, "mutators": null, "fixed_response": "Hey there", "interactive_mode": true, "parlai_home": "/private/home/komeili/dev/ParlAI", "override": {"fixed_response": "Hey there", "save_format": "conversations", "outfile": "self_chat_output"}, "starttime": "Dec01_15-25"}, "self_chat": true, "speakers": ["FixedResponseAgent_1", "FixedResponseAgent_2"], "version": "0.1"} \ No newline at end of file From 18356a1d856465465e4ffde6df64b1baecb4159e Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Wed, 1 Dec 2021 15:27:41 -0800 Subject: [PATCH 4/5] removed extra files. --- self_chat_output.html | 126 -------------------------------------- self_chat_output.jsonl | 1 - self_chat_output.metadata | 1 - 3 files changed, 128 deletions(-) delete mode 100644 self_chat_output.html delete mode 100644 self_chat_output.jsonl delete mode 100644 self_chat_output.metadata diff --git a/self_chat_output.html b/self_chat_output.html deleted file mode 100644 index 67dca77d083..00000000000 --- a/self_chat_output.html +++ /dev/null @@ -1,126 +0,0 @@ - - - - Rendered HTML - - - -
    - -
  • -
    - -
    -
    -

    Hey there

    -
    -
    -
    • - -
  • -
    - -
    -
    -

    Hey there

    -
    -
    -
    • - -
  • -
    - -
    -
    -

    Hey there

    -
    -
    -
    • - -
  • -
    - -
    -
    -

    Hey there

    -
    -
    -
    • - -
  • -
    - -
    -
    -

    Hey there

    -
    -
    -
    • - -
    • -
- - - \ No newline at end of file diff --git a/self_chat_output.jsonl b/self_chat_output.jsonl deleted file mode 100644 index 792d56320e6..00000000000 --- a/self_chat_output.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"dialog": [[{"id": "FixedResponseAgent_1", "text": "Hey there", "episode_done": false}, {"id": "FixedResponseAgent_2", "text": "Hey there", "episode_done": false}], [{"id": "FixedResponseAgent_1", "text": "Hey there", "episode_done": false}, {"id": "FixedResponseAgent_2", "text": "Hey there", "episode_done": false}], [{"id": "FixedResponseAgent_1", "text": "Hey there", "episode_done": false}, {"id": "FixedResponseAgent_2", "text": "Hey there", "episode_done": false}], [{"id": "FixedResponseAgent_1", "text": "Hey there", "episode_done": false}, {"id": "FixedResponseAgent_2", "text": "Hey there", "episode_done": false}], [{"id": "FixedResponseAgent_1", "text": "Hey there", "episode_done": false}, {"id": "FixedResponseAgent_2", "text": "Hey there", "episode_done": false}]], "context": [{"text": "Hi!", "episode_done": false, "id": "context"}, {"text": "", "episode_done": false, "id": "context"}], "metadata_path": "self_chat_output.metadata"} diff --git a/self_chat_output.metadata b/self_chat_output.metadata deleted file mode 100644 index e6f02a1ad6d..00000000000 --- a/self_chat_output.metadata +++ /dev/null @@ -1 +0,0 @@ -{"date": "2021-12-01 15:25:29.530209", "opt": {"init_opt": null, "allow_missing_init_opts": false, "task": "self_chat", "download_path": null, "loglevel": "info", "datatype": "train", "image_mode": "raw", "hide_labels": false, "multitask_weights": [1], "batchsize": 1, "dynamic_batching": null, "verbose": false, "is_debug": false, "datapath": "/private/home/komeili/dev/ParlAI/data", "model": "fixed_response", "model_file": null, "init_model": null, "dict_class": null, "seed": 42, "display_examples": true, "display_add_fields": "", "selfchat_task": true, "num_self_chats": 1, "selfchat_max_turns": 6, "seed_messages_from_task": false, "seed_messages_from_file": null, "outfile": "self_chat_output", "save_format": "conversations", "partner_model_file": null, "partner_opt_file": null, "log_keep_fields": "all", "image_size": 256, "image_cropsize": 224, "mutators": null, "fixed_response": "Hey there", "interactive_mode": true, "parlai_home": "/private/home/komeili/dev/ParlAI", "override": {"fixed_response": "Hey there", "save_format": "conversations", "outfile": "self_chat_output"}, "starttime": "Dec01_15-25"}, "self_chat": true, "speakers": ["FixedResponseAgent_1", "FixedResponseAgent_2"], "version": "0.1"} \ No newline at end of file From b6a434b79a4fbad8e0587d36a5f8b6c7621c40db Mon Sep 17 00:00:00 2001 From: Mojtaba Date: Wed, 1 Dec 2021 18:35:32 -0800 Subject: [PATCH 5/5] removed the extra condition --- parlai/utils/conversations.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/parlai/utils/conversations.py b/parlai/utils/conversations.py index 54c9142d8c4..bca8ad92397 100644 --- a/parlai/utils/conversations.py +++ b/parlai/utils/conversations.py @@ -279,11 +279,7 @@ def _get_path(datapath): @staticmethod def _check_parent_dir_exits(datapath): parent_dir = os.path.dirname(datapath) - if ( - not parent_dir - or PathManager.exists(parent_dir) - and PathManager.isdir(parent_dir) - ): + if not parent_dir or PathManager.exists(parent_dir): return logging.info(f'Parent directory ({parent_dir}) did not exist and was created.') PathManager.mkdirs(parent_dir)