From 6f12ff79c58a857b44e0b82ea87633562ce6118d Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Thu, 8 Apr 2021 11:54:21 -0700 Subject: [PATCH 1/9] Initial commit --- .../Runtime/Inference/ApplierImpl.cs | 7 +++- .../Inference/BarracudaModelExtensions.cs | 36 +++++++++---------- .../Inference/BarracudaModelParamLoader.cs | 3 +- .../Runtime/Inference/GeneratorImpl.cs | 7 ++-- .../Runtime/Inference/TensorApplier.cs | 10 +++--- .../Runtime/Inference/TensorGenerator.cs | 10 +++--- com.unity.ml-agents/package.json | 2 +- ml-agents/mlagents/trainers/torch/layers.py | 10 ++++++ ml-agents/mlagents/trainers/torch/networks.py | 1 + 9 files changed, 51 insertions(+), 35 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs b/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs index fe459ecc71..dea84d25b2 100644 --- a/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs +++ b/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs @@ -182,7 +182,7 @@ public MemoryOutputApplier( public void Apply(TensorProxy tensorProxy, IList actionIds, Dictionary lastActions) { var agentIndex = 0; - var memorySize = (int)tensorProxy.shape[tensorProxy.shape.Length - 1]; + var memorySize = tensorProxy.data.width;// (int)tensorProxy.shape[tensorProxy.shape.Length - 1]; for (var i = 0; i < actionIds.Count; i++) { var agentId = actionIds[i]; @@ -194,6 +194,11 @@ public void Apply(TensorProxy tensorProxy, IList actionIds, Dictionary GetInputTensors(this Model model) }); } - foreach (var mem in model.memories) - { - tensors.Add(new TensorProxy - { - name = mem.input, - valueType = TensorProxy.TensorType.FloatingPoint, - data = null, - shape = TensorUtils.TensorShapeFromBarracuda(mem.shape) - }); - } + // foreach (var mem in model.memories) + // { + // tensors.Add(new TensorProxy + // { + // name = mem.input, + // valueType = TensorProxy.TensorType.FloatingPoint, + // data = null, + // shape = TensorUtils.TensorShapeFromBarracuda(mem.shape) + // }); + // } tensors.Sort((el1, el2) => string.Compare(el1.name, el2.name, StringComparison.InvariantCulture)); @@ -142,14 +142,14 @@ public static string[] GetOutputNames(this Model model) names.Add(model.DiscreteOutputName()); } - var memory = (int)model.GetTensorByName(TensorNames.MemorySize)[0]; - if (memory > 0) - { - foreach (var mem in model.memories) - { - names.Add(mem.output); - } - } + // var memory = (int)model.GetTensorByName(TensorNames.MemorySize)[0]; + // if (memory > 0) + // { + // foreach (var mem in model.memories) + // { + // names.Add(mem.output); + // } + // } names.Sort(StringComparer.InvariantCulture); diff --git a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs index 149bab4b59..1348b1074f 100644 --- a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs +++ b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs @@ -289,8 +289,7 @@ ISensor[] sensors // If the model has a non-negative memory size but requires a recurrent input if (memory > 0) { - if (!tensorsNames.Any(x => x.EndsWith("_h")) || - !tensorsNames.Any(x => x.EndsWith("_c"))) + if (!tensorsNames.Any(x => x == TensorNames.RecurrentInPlaceholder)) { failedModelChecks.Add( FailedCheck.Warning("The model does not contain a Recurrent Input Node but has memory_size.") diff --git a/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs b/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs index 25c65cb87b..2736a7ad24 100644 --- a/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs +++ b/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs @@ -96,7 +96,8 @@ public void Generate( { TensorUtils.ResizeTensor(tensorProxy, batchSize, m_Allocator); - var memorySize = tensorProxy.shape[tensorProxy.shape.Length - 1]; + var memorySize = tensorProxy.data.width;//tensorProxy.shape[tensorProxy.shape.Length - 1]; + var agentIndex = 0; for (var infoIndex = 0; infoIndex < infos.Count; infoIndex++) { @@ -112,7 +113,7 @@ public void Generate( { for (var j = 0; j < memorySize; j++) { - tensorProxy.data[agentIndex, j] = 0; + tensorProxy.data[agentIndex, 0, j, 0] = 0; } agentIndex++; continue; @@ -123,7 +124,7 @@ public void Generate( { break; } - tensorProxy.data[agentIndex, j] = memory[j]; + tensorProxy.data[agentIndex, 0, j, 0] = memory[j]; } agentIndex++; } diff --git a/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs b/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs index 8845a9da30..7edd698d83 100644 --- a/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs +++ b/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs @@ -82,11 +82,11 @@ public TensorApplier( } m_Dict[TensorNames.RecurrentOutput] = new MemoryOutputApplier(memories); - for (var i = 0; i < model?.memories.Count; i++) - { - m_Dict[model.memories[i].output] = - new BarracudaMemoryOutputApplier(model.memories.Count, i, memories); - } + // for (var i = 0; i < model?.memories.Count; i++) + // { + // m_Dict[model.memories[i].output] = + // new BarracudaMemoryOutputApplier(model.memories.Count, i, memories); + // } } /// diff --git a/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs b/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs index ec51a0f58c..6890e8ad58 100644 --- a/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs +++ b/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs @@ -67,11 +67,11 @@ public TensorGenerator( m_Dict[TensorNames.RecurrentInPlaceholder] = new RecurrentInputGenerator(allocator, memories); - for (var i = 0; i < model.memories.Count; i++) - { - m_Dict[model.memories[i].input] = - new BarracudaRecurrentInputGenerator(i, allocator, memories); - } + // for (var i = 0; i < model.memories.Count; i++) + // { + // m_Dict[model.memories[i].input] = + // new BarracudaRecurrentInputGenerator(i, allocator, memories); + // } m_Dict[TensorNames.PreviousActionPlaceholder] = new PreviousActionInputGenerator(allocator); diff --git a/com.unity.ml-agents/package.json b/com.unity.ml-agents/package.json index 16865e2c91..62e16e879b 100755 --- a/com.unity.ml-agents/package.json +++ b/com.unity.ml-agents/package.json @@ -5,7 +5,7 @@ "unity": "2019.4", "description": "Use state-of-the-art machine learning to create intelligent character behaviors in any Unity environment (games, robotics, film, etc.).", "dependencies": { - "com.unity.barracuda": "1.3.2-preview", + "com.unity.barracuda": "1.3.3-preview", "com.unity.modules.imageconversion": "1.0.0", "com.unity.modules.jsonserialize": "1.0.0" } diff --git a/ml-agents/mlagents/trainers/torch/layers.py b/ml-agents/mlagents/trainers/torch/layers.py index becfb0f10d..79d6f460cc 100644 --- a/ml-agents/mlagents/trainers/torch/layers.py +++ b/ml-agents/mlagents/trainers/torch/layers.py @@ -2,6 +2,7 @@ import abc from typing import Tuple from enum import Enum +from mlagents.trainers.torch.model_serialization import exporting_to_onnx class Swish(torch.nn.Module): @@ -206,7 +207,16 @@ def forward( # We don't use torch.split here since it is not supported by Barracuda h0 = memories[:, :, : self.hidden_size].contiguous() c0 = memories[:, :, self.hidden_size :].contiguous() + + if exporting_to_onnx.is_exporting(): + h0 = torch.transpose(h0, 0, 1) + c0 = torch.transpose(c0, 0, 1) + hidden = (h0, c0) lstm_out, hidden_out = self.lstm(input_tensor, hidden) output_mem = torch.cat(hidden_out, dim=-1) + + if exporting_to_onnx.is_exporting(): + output_mem = torch.transpose(output_mem, 0, 1) + return lstm_out, output_mem diff --git a/ml-agents/mlagents/trainers/torch/networks.py b/ml-agents/mlagents/trainers/torch/networks.py index ca6c8c9481..1cef6733c1 100644 --- a/ml-agents/mlagents/trainers/torch/networks.py +++ b/ml-agents/mlagents/trainers/torch/networks.py @@ -643,6 +643,7 @@ def forward( At this moment, torch.onnx.export() doesn't accept None as tensor to be exported, so the size of return tuple varies with action spec. """ + encoding, memories_out = self.network_body( inputs, memories=memories, sequence_length=1 ) From 9e5f864c4c8b00b574b97944cbd7b4ad76de721b Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Thu, 8 Apr 2021 13:42:11 -0700 Subject: [PATCH 2/9] making it work with hallway. Added a new model version --- .../Runtime/Inference/ApplierImpl.cs | 2 +- .../Inference/BarracudaModelExtensions.cs | 48 ++++++++++++------- .../Inference/BarracudaModelParamLoader.cs | 42 ++++++++++++---- .../Runtime/Inference/TensorApplier.cs | 17 ++++--- .../Runtime/Inference/TensorGenerator.cs | 15 +++--- ml-agents/mlagents/trainers/torch/networks.py | 2 +- 6 files changed, 84 insertions(+), 42 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs b/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs index dea84d25b2..581ef102a2 100644 --- a/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs +++ b/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs @@ -182,7 +182,7 @@ public MemoryOutputApplier( public void Apply(TensorProxy tensorProxy, IList actionIds, Dictionary lastActions) { var agentIndex = 0; - var memorySize = tensorProxy.data.width;// (int)tensorProxy.shape[tensorProxy.shape.Length - 1]; + var memorySize = tensorProxy.data.width; for (var i = 0; i < actionIds.Count; i++) { var agentId = actionIds[i]; diff --git a/com.unity.ml-agents/Runtime/Inference/BarracudaModelExtensions.cs b/com.unity.ml-agents/Runtime/Inference/BarracudaModelExtensions.cs index a824552009..3908b655a6 100644 --- a/com.unity.ml-agents/Runtime/Inference/BarracudaModelExtensions.cs +++ b/com.unity.ml-agents/Runtime/Inference/BarracudaModelExtensions.cs @@ -77,16 +77,20 @@ public static IReadOnlyList GetInputTensors(this Model model) }); } - // foreach (var mem in model.memories) - // { - // tensors.Add(new TensorProxy - // { - // name = mem.input, - // valueType = TensorProxy.TensorType.FloatingPoint, - // data = null, - // shape = TensorUtils.TensorShapeFromBarracuda(mem.shape) - // }); - // } + var modelVersion = model.GetVersion(); + if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + { + foreach (var mem in model.memories) + { + tensors.Add(new TensorProxy + { + name = mem.input, + valueType = TensorProxy.TensorType.FloatingPoint, + data = null, + shape = TensorUtils.TensorShapeFromBarracuda(mem.shape) + }); + } + } tensors.Sort((el1, el2) => string.Compare(el1.name, el2.name, StringComparison.InvariantCulture)); @@ -142,14 +146,22 @@ public static string[] GetOutputNames(this Model model) names.Add(model.DiscreteOutputName()); } - // var memory = (int)model.GetTensorByName(TensorNames.MemorySize)[0]; - // if (memory > 0) - // { - // foreach (var mem in model.memories) - // { - // names.Add(mem.output); - // } - // } + var modelVersion = model.GetVersion(); + var memory = (int)model.GetTensorByName(TensorNames.MemorySize)[0]; + if (memory > 0) + { + if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + { + foreach (var mem in model.memories) + { + names.Add(mem.output); + } + } + else + { + names.Add(TensorNames.RecurrentOutput); + } + } names.Sort(StringComparer.InvariantCulture); diff --git a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs index 1348b1074f..59f5f30e28 100644 --- a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs +++ b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs @@ -19,8 +19,9 @@ internal enum ModelApiVersion { MLAgents1_0 = 2, MLAgents2_0 = 3, + MLAgents2_0_Recurrent = 4, MinSupportedVersion = MLAgents1_0, - MaxSupportedVersion = MLAgents2_0 + MaxSupportedVersion = MLAgents2_0_Recurrent } internal class FailedCheck @@ -121,7 +122,7 @@ public static IEnumerable CheckModel( CheckInputTensorShapeLegacy(model, brainParameters, sensors, observableAttributeTotalSize) ); } - else if (modelApiVersion == (int)ModelApiVersion.MLAgents2_0) + else if (modelApiVersion == (int)ModelApiVersion.MLAgents2_0 || modelApiVersion == (int)ModelApiVersion.MLAgents2_0_Recurrent) { failedModelChecks.AddRange( CheckInputTensorPresence(model, brainParameters, memorySize, sensors) @@ -289,11 +290,22 @@ ISensor[] sensors // If the model has a non-negative memory size but requires a recurrent input if (memory > 0) { - if (!tensorsNames.Any(x => x == TensorNames.RecurrentInPlaceholder)) + var modelVersion = model.GetVersion(); + var netHasMemories = false; + if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + { + netHasMemories = tensorsNames.Any(x => x.EndsWith("_h")) && + tensorsNames.Any(x => x.EndsWith("_c")); + } + else + { + netHasMemories = tensorsNames.Any(x => x == TensorNames.RecurrentInPlaceholder); + } + if (!netHasMemories) { failedModelChecks.Add( - FailedCheck.Warning("The model does not contain a Recurrent Input Node but has memory_size.") - ); + FailedCheck.Warning("The model does not contain a Recurrent Input Node but has memory_size.") + ); } } @@ -328,15 +340,27 @@ static IEnumerable CheckOutputTensorPresence(Model model, int memor // If there is no Recurrent Output but the model is Recurrent. if (memory > 0) { - var memOutputs = model.memories.Select(x => x.output).ToList(); - if (!memOutputs.Any(x => x.EndsWith("_h")) || - !memOutputs.Any(x => x.EndsWith("_c"))) + var netHasMemories = false; + var modelVersion = model.GetVersion(); + if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + { + var memOutputs = model.memories.Select(x => x.output).ToList(); + netHasMemories = memOutputs.Any(x => x.EndsWith("_h")) && + memOutputs.Any(x => x.EndsWith("_c")); + } + else + { + var allOutputs = model.GetOutputNames().ToList(); + netHasMemories = allOutputs.Any(x => x == TensorNames.RecurrentOutput); + } + if (!netHasMemories) { failedModelChecks.Add( FailedCheck.Warning("The model does not contain a Recurrent Output Node but has memory_size.") ); } + } return failedModelChecks; } @@ -717,7 +741,7 @@ static IEnumerable CheckOutputTensorShape( var modelSumDiscreteBranchSizes = model.DiscreteOutputSize(); discreteError = CheckDiscreteActionOutputShapeLegacy(brainParameters, actuatorComponents, modelSumDiscreteBranchSizes); } - if (modelApiVersion == (int)ModelApiVersion.MLAgents2_0) + if (modelApiVersion == (int)ModelApiVersion.MLAgents2_0 || modelApiVersion == (int)ModelApiVersion.MLAgents2_0_Recurrent) { var modelDiscreteBranches = model.GetTensorByName(TensorNames.DiscreteActionOutputShape); discreteError = CheckDiscreteActionOutputShape(brainParameters, actuatorComponents, modelDiscreteBranches); diff --git a/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs b/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs index 7edd698d83..5a97372559 100644 --- a/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs +++ b/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs @@ -67,26 +67,29 @@ public TensorApplier( var tensorName = model.ContinuousOutputName(); m_Dict[tensorName] = new ContinuousActionOutputApplier(actionSpec); } + var modelVersion = model.GetVersion(); if (actionSpec.NumDiscreteActions > 0) { var tensorName = model.DiscreteOutputName(); - var modelVersion = model.GetVersion(); if (modelVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents1_0) { m_Dict[tensorName] = new LegacyDiscreteActionOutputApplier(actionSpec, seed, allocator); } - if (modelVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0) + if (modelVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0 || modelVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) { m_Dict[tensorName] = new DiscreteActionOutputApplier(actionSpec, seed, allocator); } } m_Dict[TensorNames.RecurrentOutput] = new MemoryOutputApplier(memories); - // for (var i = 0; i < model?.memories.Count; i++) - // { - // m_Dict[model.memories[i].output] = - // new BarracudaMemoryOutputApplier(model.memories.Count, i, memories); - // } + if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + { + for (var i = 0; i < model?.memories.Count; i++) + { + m_Dict[model.memories[i].output] = + new BarracudaMemoryOutputApplier(model.memories.Count, i, memories); + } + } } /// diff --git a/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs b/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs index 6890e8ad58..16d2757576 100644 --- a/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs +++ b/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs @@ -67,11 +67,14 @@ public TensorGenerator( m_Dict[TensorNames.RecurrentInPlaceholder] = new RecurrentInputGenerator(allocator, memories); - // for (var i = 0; i < model.memories.Count; i++) - // { - // m_Dict[model.memories[i].input] = - // new BarracudaRecurrentInputGenerator(i, allocator, memories); - // } + if (m_ApiVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + { + for (var i = 0; i < model.memories.Count; i++) + { + m_Dict[model.memories[i].input] = + new BarracudaRecurrentInputGenerator(i, allocator, memories); + } + } m_Dict[TensorNames.PreviousActionPlaceholder] = new PreviousActionInputGenerator(allocator); @@ -141,7 +144,7 @@ public void InitializeObservations(List sensors, ITensorAllocator alloc } } - if (m_ApiVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0) + if (m_ApiVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0 || m_ApiVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) { for (var sensorIndex = 0; sensorIndex < sensors.Count; sensorIndex++) { diff --git a/ml-agents/mlagents/trainers/torch/networks.py b/ml-agents/mlagents/trainers/torch/networks.py index 1cef6733c1..c945499725 100644 --- a/ml-agents/mlagents/trainers/torch/networks.py +++ b/ml-agents/mlagents/trainers/torch/networks.py @@ -546,7 +546,7 @@ def forward( class SimpleActor(nn.Module, Actor): - MODEL_EXPORT_VERSION = 3 + MODEL_EXPORT_VERSION = 4 def __init__( self, From a541af966fc4bdb1eb95beee70c9a863ea4330b1 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Fri, 9 Apr 2021 10:03:58 -0700 Subject: [PATCH 3/9] addressing feedback --- .../Runtime/Inference/BarracudaModelExtensions.cs | 4 ++-- .../Runtime/Inference/BarracudaModelParamLoader.cs | 11 +++++------ .../Runtime/Inference/TensorApplier.cs | 4 ++-- .../Runtime/Inference/TensorGenerator.cs | 4 ++-- ml-agents/mlagents/trainers/torch/networks.py | 2 +- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Inference/BarracudaModelExtensions.cs b/com.unity.ml-agents/Runtime/Inference/BarracudaModelExtensions.cs index 3908b655a6..fc2e3a0168 100644 --- a/com.unity.ml-agents/Runtime/Inference/BarracudaModelExtensions.cs +++ b/com.unity.ml-agents/Runtime/Inference/BarracudaModelExtensions.cs @@ -78,7 +78,7 @@ public static IReadOnlyList GetInputTensors(this Model model) } var modelVersion = model.GetVersion(); - if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0) { foreach (var mem in model.memories) { @@ -150,7 +150,7 @@ public static string[] GetOutputNames(this Model model) var memory = (int)model.GetTensorByName(TensorNames.MemorySize)[0]; if (memory > 0) { - if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0) { foreach (var mem in model.memories) { diff --git a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs index 59f5f30e28..151efed169 100644 --- a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs +++ b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs @@ -19,9 +19,8 @@ internal enum ModelApiVersion { MLAgents1_0 = 2, MLAgents2_0 = 3, - MLAgents2_0_Recurrent = 4, MinSupportedVersion = MLAgents1_0, - MaxSupportedVersion = MLAgents2_0_Recurrent + MaxSupportedVersion = MLAgents2_0 } internal class FailedCheck @@ -122,7 +121,7 @@ public static IEnumerable CheckModel( CheckInputTensorShapeLegacy(model, brainParameters, sensors, observableAttributeTotalSize) ); } - else if (modelApiVersion == (int)ModelApiVersion.MLAgents2_0 || modelApiVersion == (int)ModelApiVersion.MLAgents2_0_Recurrent) + else if (modelApiVersion == (int)ModelApiVersion.MLAgents2_0) { failedModelChecks.AddRange( CheckInputTensorPresence(model, brainParameters, memorySize, sensors) @@ -292,7 +291,7 @@ ISensor[] sensors { var modelVersion = model.GetVersion(); var netHasMemories = false; - if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0) { netHasMemories = tensorsNames.Any(x => x.EndsWith("_h")) && tensorsNames.Any(x => x.EndsWith("_c")); @@ -343,7 +342,7 @@ static IEnumerable CheckOutputTensorPresence(Model model, int memor var netHasMemories = false; var modelVersion = model.GetVersion(); - if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0) { var memOutputs = model.memories.Select(x => x.output).ToList(); netHasMemories = memOutputs.Any(x => x.EndsWith("_h")) && @@ -741,7 +740,7 @@ static IEnumerable CheckOutputTensorShape( var modelSumDiscreteBranchSizes = model.DiscreteOutputSize(); discreteError = CheckDiscreteActionOutputShapeLegacy(brainParameters, actuatorComponents, modelSumDiscreteBranchSizes); } - if (modelApiVersion == (int)ModelApiVersion.MLAgents2_0 || modelApiVersion == (int)ModelApiVersion.MLAgents2_0_Recurrent) + if (modelApiVersion == (int)ModelApiVersion.MLAgents2_0) { var modelDiscreteBranches = model.GetTensorByName(TensorNames.DiscreteActionOutputShape); discreteError = CheckDiscreteActionOutputShape(brainParameters, actuatorComponents, modelDiscreteBranches); diff --git a/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs b/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs index 5a97372559..b9487bbc70 100644 --- a/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs +++ b/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs @@ -75,14 +75,14 @@ public TensorApplier( { m_Dict[tensorName] = new LegacyDiscreteActionOutputApplier(actionSpec, seed, allocator); } - if (modelVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0 || modelVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + if (modelVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0) { m_Dict[tensorName] = new DiscreteActionOutputApplier(actionSpec, seed, allocator); } } m_Dict[TensorNames.RecurrentOutput] = new MemoryOutputApplier(memories); - if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + if (modelVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0) { for (var i = 0; i < model?.memories.Count; i++) { diff --git a/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs b/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs index 16d2757576..5eb8706013 100644 --- a/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs +++ b/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs @@ -67,7 +67,7 @@ public TensorGenerator( m_Dict[TensorNames.RecurrentInPlaceholder] = new RecurrentInputGenerator(allocator, memories); - if (m_ApiVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + if (m_ApiVersion < (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0) { for (var i = 0; i < model.memories.Count; i++) { @@ -144,7 +144,7 @@ public void InitializeObservations(List sensors, ITensorAllocator alloc } } - if (m_ApiVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0 || m_ApiVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0_Recurrent) + if (m_ApiVersion == (int)BarracudaModelParamLoader.ModelApiVersion.MLAgents2_0) { for (var sensorIndex = 0; sensorIndex < sensors.Count; sensorIndex++) { diff --git a/ml-agents/mlagents/trainers/torch/networks.py b/ml-agents/mlagents/trainers/torch/networks.py index c945499725..e60f5737cb 100644 --- a/ml-agents/mlagents/trainers/torch/networks.py +++ b/ml-agents/mlagents/trainers/torch/networks.py @@ -546,7 +546,7 @@ def forward( class SimpleActor(nn.Module, Actor): - MODEL_EXPORT_VERSION = 4 + MODEL_EXPORT_VERSION = 3 # Corresponds to ModelApiVersion.MLAgents2_0 def __init__( self, From e24d0c996a35879a99f3382e2f7a17ea45651818 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Fri, 9 Apr 2021 14:21:49 -0700 Subject: [PATCH 4/9] Adding a comment --- ml-agents/mlagents/trainers/torch/layers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ml-agents/mlagents/trainers/torch/layers.py b/ml-agents/mlagents/trainers/torch/layers.py index 79d6f460cc..c4475d5f95 100644 --- a/ml-agents/mlagents/trainers/torch/layers.py +++ b/ml-agents/mlagents/trainers/torch/layers.py @@ -209,6 +209,9 @@ def forward( c0 = memories[:, :, self.hidden_size :].contiguous() if exporting_to_onnx.is_exporting(): + # This transpose is needed both at input and output of the LSTM when + # exporting because ONNX will expect (sequence_len, batch, memory_size) + # instead of (batch, sequence_len, memory_size) h0 = torch.transpose(h0, 0, 1) c0 = torch.transpose(c0, 0, 1) From 9e19e80184c8ffde3dd8287d8b485d0d11a35ea1 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Fri, 9 Apr 2021 14:22:58 -0700 Subject: [PATCH 5/9] formatting --- ml-agents/mlagents/trainers/torch/networks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml-agents/mlagents/trainers/torch/networks.py b/ml-agents/mlagents/trainers/torch/networks.py index e60f5737cb..25950b7ede 100644 --- a/ml-agents/mlagents/trainers/torch/networks.py +++ b/ml-agents/mlagents/trainers/torch/networks.py @@ -546,7 +546,7 @@ def forward( class SimpleActor(nn.Module, Actor): - MODEL_EXPORT_VERSION = 3 # Corresponds to ModelApiVersion.MLAgents2_0 + MODEL_EXPORT_VERSION = 3 # Corresponds to ModelApiVersion.MLAgents2_0 def __init__( self, From ca70a0b0300ef3caa3cae2dfd8b94bd756828b59 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Fri, 9 Apr 2021 15:51:08 -0700 Subject: [PATCH 6/9] modifying the changelog --- com.unity.ml-agents/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/com.unity.ml-agents/CHANGELOG.md b/com.unity.ml-agents/CHANGELOG.md index 265f2bed2f..330395ff49 100755 --- a/com.unity.ml-agents/CHANGELOG.md +++ b/com.unity.ml-agents/CHANGELOG.md @@ -29,6 +29,7 @@ produces two `Match3Sensor`s (unless there are no special types). Previously tra sizes and will need to be retrained. (#5181) - The `AbstractBoard` class for integration with Match-3 games was changed to make it easier to support boards with different sizes using the same model. For a summary of the interface changes, please see the Migration Guide. (##5189) +- Updated the Barracuda package to version `1.3.3-preview`(#5236) #### ml-agents / ml-agents-envs / gym-unity (Python) - The `--resume` flag now supports resuming experiments with additional reward providers or @@ -38,7 +39,7 @@ different sizes using the same model. For a summary of the interface changes, pl ### Minor Changes #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#) -- The `.onnx` models input names have changed. All input placeholders will now use the prefix `obs_` removing the distinction between visual and vector observations. Models created with this version will not be usable with previous versions of the package (#5080) +- The `.onnx` models input names have changed. All input placeholders will now use the prefix `obs_` removing the distinction between visual and vector observations. Models created with this version will not be usable with previous versions of the package (#5080, #5236) - The `.onnx` models discrete action output now contains the discrete actions values and not the logits. Models created with this version will not be usable with previous versions of the package (#5080) - Added ML-Agents package settings. (#5027) - Make com.unity.modules.unityanalytics an optional dependency. (#5109) From f7919e81662891d28262f7dffc45327c0bbe9e26 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Fri, 9 Apr 2021 15:59:35 -0700 Subject: [PATCH 7/9] Adding descriptions on the model version descriptions --- .../Runtime/Inference/BarracudaModelParamLoader.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs index 151efed169..383e674675 100644 --- a/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs +++ b/com.unity.ml-agents/Runtime/Inference/BarracudaModelParamLoader.cs @@ -17,7 +17,21 @@ internal class BarracudaModelParamLoader internal enum ModelApiVersion { + /// + /// ML-Agents model version for versions 1.x.y + /// The observations are split between vector and visual observations + /// There are legacy action outputs for discrete and continuous actions + /// LSTM inputs and outputs are handled by Barracuda + /// MLAgents1_0 = 2, + + /// + /// All observations are treated the same and named obs_{i} with i being + /// the sensor index + /// Legacy "action" output is no longer present + /// LSTM inputs and outputs are treated like regular inputs and outputs + /// and no longer managed by Barracuda + /// MLAgents2_0 = 3, MinSupportedVersion = MLAgents1_0, MaxSupportedVersion = MLAgents2_0 From bfedf6a90bb56139067d4ae9f417effb3fa76974 Mon Sep 17 00:00:00 2001 From: Vincent-Pierre BERGES Date: Mon, 12 Apr 2021 17:17:56 -0700 Subject: [PATCH 8/9] Update com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs Co-authored-by: Chris Elion --- com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs b/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs index 2736a7ad24..2d290538b3 100644 --- a/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs +++ b/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs @@ -96,7 +96,7 @@ public void Generate( { TensorUtils.ResizeTensor(tensorProxy, batchSize, m_Allocator); - var memorySize = tensorProxy.data.width;//tensorProxy.shape[tensorProxy.shape.Length - 1]; + var memorySize = tensorProxy.data.width; var agentIndex = 0; for (var infoIndex = 0; infoIndex < infos.Count; infoIndex++) From 65c19a7a83522477fe56dcd0fcf9ef126a9a08d6 Mon Sep 17 00:00:00 2001 From: vincentpierre Date: Tue, 13 Apr 2021 09:28:14 -0700 Subject: [PATCH 9/9] addressing comments --- com.unity.ml-agents/CHANGELOG.md | 2 +- ml-agents/mlagents/trainers/torch/layers.py | 10 +++++----- ml-agents/mlagents/trainers/torch/networks.py | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/com.unity.ml-agents/CHANGELOG.md b/com.unity.ml-agents/CHANGELOG.md index 330395ff49..ee2be47606 100755 --- a/com.unity.ml-agents/CHANGELOG.md +++ b/com.unity.ml-agents/CHANGELOG.md @@ -39,7 +39,7 @@ different sizes using the same model. For a summary of the interface changes, pl ### Minor Changes #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#) -- The `.onnx` models input names have changed. All input placeholders will now use the prefix `obs_` removing the distinction between visual and vector observations. Models created with this version will not be usable with previous versions of the package (#5080, #5236) +- The `.onnx` models input names have changed. All input placeholders will now use the prefix `obs_` removing the distinction between visual and vector observations. In addition, the inputs and outputs of LSTM changed. Models created with this version will not be usable with previous versions of the package (#5080, #5236) - The `.onnx` models discrete action output now contains the discrete actions values and not the logits. Models created with this version will not be usable with previous versions of the package (#5080) - Added ML-Agents package settings. (#5027) - Make com.unity.modules.unityanalytics an optional dependency. (#5109) diff --git a/ml-agents/mlagents/trainers/torch/layers.py b/ml-agents/mlagents/trainers/torch/layers.py index c4475d5f95..5edf3acbdf 100644 --- a/ml-agents/mlagents/trainers/torch/layers.py +++ b/ml-agents/mlagents/trainers/torch/layers.py @@ -204,16 +204,16 @@ def memory_size(self) -> int: def forward( self, input_tensor: torch.Tensor, memories: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: - # We don't use torch.split here since it is not supported by Barracuda - h0 = memories[:, :, : self.hidden_size].contiguous() - c0 = memories[:, :, self.hidden_size :].contiguous() if exporting_to_onnx.is_exporting(): # This transpose is needed both at input and output of the LSTM when # exporting because ONNX will expect (sequence_len, batch, memory_size) # instead of (batch, sequence_len, memory_size) - h0 = torch.transpose(h0, 0, 1) - c0 = torch.transpose(c0, 0, 1) + memories = torch.transpose(memories, 0, 1) + + # We don't use torch.split here since it is not supported by Barracuda + h0 = memories[:, :, : self.hidden_size].contiguous() + c0 = memories[:, :, self.hidden_size :].contiguous() hidden = (h0, c0) lstm_out, hidden_out = self.lstm(input_tensor, hidden) diff --git a/ml-agents/mlagents/trainers/torch/networks.py b/ml-agents/mlagents/trainers/torch/networks.py index 25950b7ede..0023653170 100644 --- a/ml-agents/mlagents/trainers/torch/networks.py +++ b/ml-agents/mlagents/trainers/torch/networks.py @@ -643,7 +643,6 @@ def forward( At this moment, torch.onnx.export() doesn't accept None as tensor to be exported, so the size of return tuple varies with action spec. """ - encoding, memories_out = self.network_body( inputs, memories=memories, sequence_length=1 )