From 9c0fa10fa5466db94d8e0def19e3de832069a443 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Tue, 24 Aug 2021 15:05:45 -0700 Subject: [PATCH 01/18] Custom trainer editor analytics --- ml-agents-envs/mlagents_envs/environment.py | 15 ++++++ ...default_training_analytics_side_channel.py | 49 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py diff --git a/ml-agents-envs/mlagents_envs/environment.py b/ml-agents-envs/mlagents_envs/environment.py index 88dc254b8f..8d7257597c 100644 --- a/ml-agents-envs/mlagents_envs/environment.py +++ b/ml-agents-envs/mlagents_envs/environment.py @@ -10,6 +10,9 @@ from mlagents_envs.logging_util import get_logger from mlagents_envs.side_channel.side_channel import SideChannel +from mlagents_envs.side_channel.default_training_analytics_side_channel import ( + DefaultTrainingAnalyticsSideChannel, +) from mlagents_envs.side_channel.side_channel_manager import SideChannelManager from mlagents_envs import env_utils @@ -186,6 +189,16 @@ def __init__( self._timeout_wait: int = timeout_wait self._communicator = self._get_communicator(worker_id, base_port, timeout_wait) self._worker_id = worker_id + if side_channels is None: + side_channels = [] + default_training_side_channel: Optional[ + DefaultTrainingAnalyticsSideChannel + ] = None + if "TrainingAnalyticsSideChannel" not in [ + x.__class__.__name__ for x in side_channels + ]: + default_training_side_channel = DefaultTrainingAnalyticsSideChannel() + side_channels.append(default_training_side_channel) self._side_channel_manager = SideChannelManager(side_channels) self._log_folder = log_folder self.academy_capabilities: UnityRLCapabilitiesProto = None # type: ignore @@ -246,6 +259,8 @@ def __init__( self._is_first_message = True self._update_behavior_specs(aca_output) self.academy_capabilities = aca_params.capabilities + if default_training_side_channel is not None: + default_training_side_channel.environment_initialized() @staticmethod def _get_communicator(worker_id, base_port, timeout_wait): diff --git a/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py b/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py new file mode 100644 index 0000000000..ecf2b8e291 --- /dev/null +++ b/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py @@ -0,0 +1,49 @@ +import sys +import uuid +import mlagents_envs + +from mlagents_envs.exception import UnityCommunicationException +from mlagents_envs.side_channel import SideChannel, IncomingMessage, OutgoingMessage +from mlagents_envs.communicator_objects.training_analytics_pb2 import ( + TrainingEnvironmentInitialized, +) +from google.protobuf.any_pb2 import Any + + +class DefaultTrainingAnalyticsSideChannel(SideChannel): + """ + Side channel that sends information about the training to the Unity environment so it can be logged. + """ + + def __init__(self) -> None: + # >>> uuid.uuid5(uuid.NAMESPACE_URL, "com.unity.ml-agents/TrainingAnalyticsSideChannel") + # UUID('b664a4a9-d86f-5a5f-95cb-e8353a7e8356') + # We purposefully use the SAME side channel as the TrainingAnalyticsSideChannel + super().__init__(uuid.UUID("b664a4a9-d86f-5a5f-95cb-e8353a7e8356")) + + def on_message_received(self, msg: IncomingMessage) -> None: + raise UnityCommunicationException( + "The DefaultTrainingAnalyticsSideChannel received a message from Unity, " + + "this should not have happened." + ) + + def environment_initialized(self) -> None: + # Tuple of (major, minor, patch) + vi = sys.version_info + + msg = TrainingEnvironmentInitialized( + python_version=f"{vi[0]}.{vi[1]}.{vi[2]}", + mlagents_version="Custom", + mlagents_envs_version=mlagents_envs.__version__, + torch_version="Unknown", + torch_device_type="Unknown", + num_envs=0, + num_environment_parameters=0, + ) + + any_message = Any() + any_message.Pack(msg) + + env_init_msg = OutgoingMessage() + env_init_msg.set_raw_bytes(any_message.SerializeToString()) # type: ignore + super().queue_message_to_send(env_init_msg) From 0a558c61ce3f40b4a8d9886ce8c4dbb3c2eb9997 Mon Sep 17 00:00:00 2001 From: mahon94 Date: Mon, 30 Aug 2021 09:44:13 -0700 Subject: [PATCH 02/18] inherit the default_training_analytics_side_channel --- ml-agents-envs/mlagents_envs/environment.py | 4 +--- .../mlagents_envs/side_channel/__init__.py | 3 +++ .../default_training_analytics_side_channel.py | 1 - .../training_analytics_side_channel.py | 18 +++++++++--------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ml-agents-envs/mlagents_envs/environment.py b/ml-agents-envs/mlagents_envs/environment.py index 8d7257597c..7a3ca0c2f6 100644 --- a/ml-agents-envs/mlagents_envs/environment.py +++ b/ml-agents-envs/mlagents_envs/environment.py @@ -10,9 +10,7 @@ from mlagents_envs.logging_util import get_logger from mlagents_envs.side_channel.side_channel import SideChannel -from mlagents_envs.side_channel.default_training_analytics_side_channel import ( - DefaultTrainingAnalyticsSideChannel, -) +from mlagents_envs.side_channel import DefaultTrainingAnalyticsSideChannel from mlagents_envs.side_channel.side_channel_manager import SideChannelManager from mlagents_envs import env_utils diff --git a/ml-agents-envs/mlagents_envs/side_channel/__init__.py b/ml-agents-envs/mlagents_envs/side_channel/__init__.py index d6ccfdf573..0da4521c44 100644 --- a/ml-agents-envs/mlagents_envs/side_channel/__init__.py +++ b/ml-agents-envs/mlagents_envs/side_channel/__init__.py @@ -2,3 +2,6 @@ from mlagents_envs.side_channel.outgoing_message import OutgoingMessage # noqa from mlagents_envs.side_channel.side_channel import SideChannel # noqa +from mlagents_envs.side_channel.default_training_analytics_side_channel import ( + DefaultTrainingAnalyticsSideChannel, +) # noqa diff --git a/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py b/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py index ecf2b8e291..92f836c85e 100644 --- a/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py +++ b/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py @@ -40,7 +40,6 @@ def environment_initialized(self) -> None: num_envs=0, num_environment_parameters=0, ) - any_message = Any() any_message.Pack(msg) diff --git a/ml-agents/mlagents/training_analytics_side_channel.py b/ml-agents/mlagents/training_analytics_side_channel.py index f964f13fac..4f43457ff0 100644 --- a/ml-agents/mlagents/training_analytics_side_channel.py +++ b/ml-agents/mlagents/training_analytics_side_channel.py @@ -6,7 +6,12 @@ from mlagents import torch_utils from mlagents.trainers.settings import RewardSignalType from mlagents_envs.exception import UnityCommunicationException -from mlagents_envs.side_channel import SideChannel, IncomingMessage, OutgoingMessage +from mlagents_envs.side_channel import ( + SideChannel, + IncomingMessage, + OutgoingMessage, + DefaultTrainingAnalyticsSideChannel, +) from mlagents_envs.communicator_objects.training_analytics_pb2 import ( TrainingEnvironmentInitialized, TrainingBehaviorInitialized, @@ -16,7 +21,7 @@ from mlagents.trainers.settings import TrainerSettings, RunOptions -class TrainingAnalyticsSideChannel(SideChannel): +class TrainingAnalyticsSideChannel(DefaultTrainingAnalyticsSideChannel): """ Side channel that sends information about the training to the Unity environment so it can be logged. """ @@ -24,15 +29,10 @@ class TrainingAnalyticsSideChannel(SideChannel): def __init__(self) -> None: # >>> uuid.uuid5(uuid.NAMESPACE_URL, "com.unity.ml-agents/TrainingAnalyticsSideChannel") # UUID('b664a4a9-d86f-5a5f-95cb-e8353a7e8356') - super().__init__(uuid.UUID("b664a4a9-d86f-5a5f-95cb-e8353a7e8356")) + # Use the same uuid as the parent side channel + super().__init__() self.run_options: Optional[RunOptions] = None - def on_message_received(self, msg: IncomingMessage) -> None: - raise UnityCommunicationException( - "The TrainingAnalyticsSideChannel received a message from Unity, " - + "this should not have happened." - ) - def environment_initialized(self, run_options: RunOptions) -> None: self.run_options = run_options # Tuple of (major, minor, patch) From 291b68081e041fc8102c6d2118059d3f67388593 Mon Sep 17 00:00:00 2001 From: mahon94 Date: Mon, 30 Aug 2021 15:41:10 -0700 Subject: [PATCH 03/18] inherit the default_training_analytics_side_channel --- ml-agents-envs/mlagents_envs/environment.py | 4 +++- ml-agents-envs/mlagents_envs/side_channel/__init__.py | 3 --- ml-agents/mlagents/training_analytics_side_channel.py | 8 ++++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ml-agents-envs/mlagents_envs/environment.py b/ml-agents-envs/mlagents_envs/environment.py index 7a3ca0c2f6..8d7257597c 100644 --- a/ml-agents-envs/mlagents_envs/environment.py +++ b/ml-agents-envs/mlagents_envs/environment.py @@ -10,7 +10,9 @@ from mlagents_envs.logging_util import get_logger from mlagents_envs.side_channel.side_channel import SideChannel -from mlagents_envs.side_channel import DefaultTrainingAnalyticsSideChannel +from mlagents_envs.side_channel.default_training_analytics_side_channel import ( + DefaultTrainingAnalyticsSideChannel, +) from mlagents_envs.side_channel.side_channel_manager import SideChannelManager from mlagents_envs import env_utils diff --git a/ml-agents-envs/mlagents_envs/side_channel/__init__.py b/ml-agents-envs/mlagents_envs/side_channel/__init__.py index 0da4521c44..d6ccfdf573 100644 --- a/ml-agents-envs/mlagents_envs/side_channel/__init__.py +++ b/ml-agents-envs/mlagents_envs/side_channel/__init__.py @@ -2,6 +2,3 @@ from mlagents_envs.side_channel.outgoing_message import OutgoingMessage # noqa from mlagents_envs.side_channel.side_channel import SideChannel # noqa -from mlagents_envs.side_channel.default_training_analytics_side_channel import ( - DefaultTrainingAnalyticsSideChannel, -) # noqa diff --git a/ml-agents/mlagents/training_analytics_side_channel.py b/ml-agents/mlagents/training_analytics_side_channel.py index 4f43457ff0..4b1f1c2dd5 100644 --- a/ml-agents/mlagents/training_analytics_side_channel.py +++ b/ml-agents/mlagents/training_analytics_side_channel.py @@ -1,13 +1,11 @@ import sys from typing import Optional -import uuid import mlagents_envs import mlagents.trainers from mlagents import torch_utils from mlagents.trainers.settings import RewardSignalType from mlagents_envs.exception import UnityCommunicationException from mlagents_envs.side_channel import ( - SideChannel, IncomingMessage, OutgoingMessage, DefaultTrainingAnalyticsSideChannel, @@ -33,6 +31,12 @@ def __init__(self) -> None: super().__init__() self.run_options: Optional[RunOptions] = None + def on_message_received(self, msg: IncomingMessage) -> None: + raise UnityCommunicationException( + "The TrainingAnalyticsSideChannel received a message from Unity, " + "this should not have happened." + ) + def environment_initialized(self, run_options: RunOptions) -> None: self.run_options = run_options # Tuple of (major, minor, patch) From d744766f16a6c05ba22b4b35b1655cc7bcae5620 Mon Sep 17 00:00:00 2001 From: mahon94 Date: Mon, 30 Aug 2021 16:53:56 -0700 Subject: [PATCH 04/18] fixing pytest --- ml-agents-envs/mlagents_envs/environment.py | 4 +--- ml-agents-envs/mlagents_envs/side_channel/__init__.py | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ml-agents-envs/mlagents_envs/environment.py b/ml-agents-envs/mlagents_envs/environment.py index 8d7257597c..7a3ca0c2f6 100644 --- a/ml-agents-envs/mlagents_envs/environment.py +++ b/ml-agents-envs/mlagents_envs/environment.py @@ -10,9 +10,7 @@ from mlagents_envs.logging_util import get_logger from mlagents_envs.side_channel.side_channel import SideChannel -from mlagents_envs.side_channel.default_training_analytics_side_channel import ( - DefaultTrainingAnalyticsSideChannel, -) +from mlagents_envs.side_channel import DefaultTrainingAnalyticsSideChannel from mlagents_envs.side_channel.side_channel_manager import SideChannelManager from mlagents_envs import env_utils diff --git a/ml-agents-envs/mlagents_envs/side_channel/__init__.py b/ml-agents-envs/mlagents_envs/side_channel/__init__.py index d6ccfdf573..c9a1f5f0f7 100644 --- a/ml-agents-envs/mlagents_envs/side_channel/__init__.py +++ b/ml-agents-envs/mlagents_envs/side_channel/__init__.py @@ -2,3 +2,6 @@ from mlagents_envs.side_channel.outgoing_message import OutgoingMessage # noqa from mlagents_envs.side_channel.side_channel import SideChannel # noqa +from mlagents_envs.side_channel.default_training_analytics_side_channel import ( # noqa + DefaultTrainingAnalyticsSideChannel, # noqa +) # noqa From abad5758f01330429d78c67bd4ccc0d3c5a00ad3 Mon Sep 17 00:00:00 2001 From: mahon94 Date: Thu, 2 Sep 2021 09:37:27 -0700 Subject: [PATCH 05/18] find channel with uuid, remove unused attr in proto --- ml-agents-envs/mlagents_envs/environment.py | 5 +++-- .../default_training_analytics_side_channel.py | 9 +++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/ml-agents-envs/mlagents_envs/environment.py b/ml-agents-envs/mlagents_envs/environment.py index 7a3ca0c2f6..f704fc9533 100644 --- a/ml-agents-envs/mlagents_envs/environment.py +++ b/ml-agents-envs/mlagents_envs/environment.py @@ -192,8 +192,8 @@ def __init__( default_training_side_channel: Optional[ DefaultTrainingAnalyticsSideChannel ] = None - if "TrainingAnalyticsSideChannel" not in [ - x.__class__.__name__ for x in side_channels + if DefaultTrainingAnalyticsSideChannel.CHANNEL_ID not in [ + _.channel_id for _ in side_channels ]: default_training_side_channel = DefaultTrainingAnalyticsSideChannel() side_channels.append(default_training_side_channel) @@ -258,6 +258,7 @@ def __init__( self._update_behavior_specs(aca_output) self.academy_capabilities = aca_params.capabilities if default_training_side_channel is not None: + print("in gym test") default_training_side_channel.environment_initialized() @staticmethod diff --git a/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py b/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py index 92f836c85e..4534b771a3 100644 --- a/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py +++ b/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py @@ -14,12 +14,13 @@ class DefaultTrainingAnalyticsSideChannel(SideChannel): """ Side channel that sends information about the training to the Unity environment so it can be logged. """ - + CHANNEL_ID = uuid.UUID("b664a4a9-d86f-5a5f-95cb-e8353a7e8356") def __init__(self) -> None: # >>> uuid.uuid5(uuid.NAMESPACE_URL, "com.unity.ml-agents/TrainingAnalyticsSideChannel") # UUID('b664a4a9-d86f-5a5f-95cb-e8353a7e8356') # We purposefully use the SAME side channel as the TrainingAnalyticsSideChannel - super().__init__(uuid.UUID("b664a4a9-d86f-5a5f-95cb-e8353a7e8356")) + + super().__init__(DefaultTrainingAnalyticsSideChannel.CHANNEL_ID) def on_message_received(self, msg: IncomingMessage) -> None: raise UnityCommunicationException( @@ -35,10 +36,6 @@ def environment_initialized(self) -> None: python_version=f"{vi[0]}.{vi[1]}.{vi[2]}", mlagents_version="Custom", mlagents_envs_version=mlagents_envs.__version__, - torch_version="Unknown", - torch_device_type="Unknown", - num_envs=0, - num_environment_parameters=0, ) any_message = Any() any_message.Pack(msg) From 1da6c4391575219fd9928902d559c223d42cb75b Mon Sep 17 00:00:00 2001 From: mahon94 Date: Thu, 2 Sep 2021 09:41:52 -0700 Subject: [PATCH 06/18] formatting --- ml-agents-envs/mlagents_envs/environment.py | 1 - .../side_channel/default_training_analytics_side_channel.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ml-agents-envs/mlagents_envs/environment.py b/ml-agents-envs/mlagents_envs/environment.py index f704fc9533..776c5d1030 100644 --- a/ml-agents-envs/mlagents_envs/environment.py +++ b/ml-agents-envs/mlagents_envs/environment.py @@ -258,7 +258,6 @@ def __init__( self._update_behavior_specs(aca_output) self.academy_capabilities = aca_params.capabilities if default_training_side_channel is not None: - print("in gym test") default_training_side_channel.environment_initialized() @staticmethod diff --git a/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py b/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py index 4534b771a3..043e972412 100644 --- a/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py +++ b/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py @@ -14,7 +14,9 @@ class DefaultTrainingAnalyticsSideChannel(SideChannel): """ Side channel that sends information about the training to the Unity environment so it can be logged. """ + CHANNEL_ID = uuid.UUID("b664a4a9-d86f-5a5f-95cb-e8353a7e8356") + def __init__(self) -> None: # >>> uuid.uuid5(uuid.NAMESPACE_URL, "com.unity.ml-agents/TrainingAnalyticsSideChannel") # UUID('b664a4a9-d86f-5a5f-95cb-e8353a7e8356') From 6baf3ad56a03b0d5f24fad1e66a3b1579fda20c6 Mon Sep 17 00:00:00 2001 From: mahon94 Date: Thu, 2 Sep 2021 09:44:49 -0700 Subject: [PATCH 07/18] formatting --- .../side_channel/default_training_analytics_side_channel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py b/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py index 043e972412..a53e686709 100644 --- a/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py +++ b/ml-agents-envs/mlagents_envs/side_channel/default_training_analytics_side_channel.py @@ -38,6 +38,8 @@ def environment_initialized(self) -> None: python_version=f"{vi[0]}.{vi[1]}.{vi[2]}", mlagents_version="Custom", mlagents_envs_version=mlagents_envs.__version__, + torch_version="Unknown", + torch_device_type="Unknown", ) any_message = Any() any_message.Pack(msg) From b726104da302f1e6ebc76e45b792b08e89ca6cd3 Mon Sep 17 00:00:00 2001 From: Chris Goy Date: Fri, 27 Aug 2021 14:37:41 -0700 Subject: [PATCH 08/18] Only update the native library and the c# calls that need to be made to clean up channels. (#5283) --- .../osx/native/libgrpc_csharp_ext.x64.bundle | Bin 3280060 -> 9741920 bytes .../Runtime/Communicator/RpcCommunicator.cs | 24 +++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/com.unity.ml-agents/Plugins/ProtoBuffer/runtimes/osx/native/libgrpc_csharp_ext.x64.bundle b/com.unity.ml-agents/Plugins/ProtoBuffer/runtimes/osx/native/libgrpc_csharp_ext.x64.bundle index 58390e6cba1709d15131453c383aaeb7c99143ca..440d2b9e337498025c3568f83038e927e1ed81d7 100755 GIT binary patch literal 9741920 zcmeFadwg6~)i*wAJD5_#1f&`$NPr3fgr*2uo)Sr*BU4BqT%tw57RseO+)5$Zs=