diff --git a/Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/SensorBase.cs b/Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/SensorBase.cs index 86f5d46d21..eed6b7a282 100644 --- a/Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/SensorBase.cs +++ b/Project/Assets/ML-Agents/Examples/SharedAssets/Scripts/SensorBase.cs @@ -33,7 +33,7 @@ public virtual int Write(ObservationWriter writer) float[] buffer = new float[numFloats]; WriteObservation(buffer); - writer.AddRange(buffer); + writer.AddList(buffer); return numFloats; } diff --git a/com.unity.ml-agents/CHANGELOG.md b/com.unity.ml-agents/CHANGELOG.md index b74c70411b..236111ad25 100755 --- a/com.unity.ml-agents/CHANGELOG.md +++ b/com.unity.ml-agents/CHANGELOG.md @@ -23,6 +23,11 @@ removed when training with a player. The Editor still requires it to be clamped - Added the IHeuristicProvider interface to allow IActuators as well as Agent implement the Heuristic function to generate actions. Updated the Basic example and the Match3 Example to use Actuators. Changed the namespace and file names of classes in com.unity.ml-agents.extensions. (#4849) +- Added `VectorSensor.AddObservation(IList)`. `VectorSensor.AddObservation(IEnumerable)` + is deprecated. The `IList` version is recommended, as it does not generate any + additional memory allocations. (#4887) +- Added `ObservationWriter.AddList()` and deprecated `ObservationWriter.AddRange()`. + `AddList()` is recommended, as it does not generate any additional memory allocations. (#4887) #### ml-agents / ml-agents-envs / gym-unity (Python) @@ -30,6 +35,9 @@ removed when training with a player. The Editor still requires it to be clamped #### com.unity.ml-agents (C#) - Fix a compile warning about using an obsolete enum in `GrpcExtensions.cs`. (#4812) - CameraSensor now logs an error if the GraphicsDevice is null. (#4880) +- Removed several memory allocations that happened during inference. On a test scene, this + reduced the amount of memory allocated by approximately 25%. (#4887) + #### ml-agents / ml-agents-envs / gym-unity (Python) - Fixed a bug that would cause an exception when `RunOptions` was deserialized via `pickle`. (#4842) - Fixed a bug that can cause a crash if a behavior can appear during training in multi-environment training. (#4872) diff --git a/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs b/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs index 6675545d4b..3952786306 100644 --- a/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs +++ b/com.unity.ml-agents/Runtime/Inference/ApplierImpl.cs @@ -21,12 +21,13 @@ public ContinuousActionOutputApplier(ActionSpec actionSpec) m_ActionSpec = actionSpec; } - public void Apply(TensorProxy tensorProxy, IEnumerable actionIds, Dictionary lastActions) + public void Apply(TensorProxy tensorProxy, IList actionIds, Dictionary lastActions) { var actionSize = tensorProxy.shape[tensorProxy.shape.Length - 1]; var agentIndex = 0; - foreach (int agentId in actionIds) + for (var i = 0; i < actionIds.Count; i++) { + var agentId = actionIds[i]; if (lastActions.ContainsKey(agentId)) { var actionBuffer = lastActions[agentId]; @@ -65,7 +66,7 @@ public DiscreteActionOutputApplier(ActionSpec actionSpec, int seed, ITensorAlloc m_ActionSpec = actionSpec; } - public void Apply(TensorProxy tensorProxy, IEnumerable actionIds, Dictionary lastActions) + public void Apply(TensorProxy tensorProxy, IList actionIds, Dictionary lastActions) { //var tensorDataProbabilities = tensorProxy.Data as float[,]; var idActionPairList = actionIds as List ?? actionIds.ToList(); @@ -109,9 +110,11 @@ public void Apply(TensorProxy tensorProxy, IEnumerable actionIds, Dictionar actionProbs.data.Dispose(); outputTensor.data.Dispose(); } + var agentIndex = 0; - foreach (int agentId in actionIds) + for (var i = 0; i < actionIds.Count; i++) { + var agentId = actionIds[i]; if (lastActions.ContainsKey(agentId)) { var actionBuffer = lastActions[agentId]; @@ -209,12 +212,13 @@ public MemoryOutputApplier( m_Memories = memories; } - public void Apply(TensorProxy tensorProxy, IEnumerable actionIds, Dictionary lastActions) + public void Apply(TensorProxy tensorProxy, IList actionIds, Dictionary lastActions) { var agentIndex = 0; var memorySize = (int)tensorProxy.shape[tensorProxy.shape.Length - 1]; - foreach (int agentId in actionIds) + for (var i = 0; i < actionIds.Count; i++) { + var agentId = actionIds[i]; List memory; if (!m_Memories.TryGetValue(agentId, out memory) || memory.Count < memorySize) @@ -246,13 +250,14 @@ public BarracudaMemoryOutputApplier( m_Memories = memories; } - public void Apply(TensorProxy tensorProxy, IEnumerable actionIds, Dictionary lastActions) + public void Apply(TensorProxy tensorProxy, IList actionIds, Dictionary lastActions) { var agentIndex = 0; var memorySize = (int)tensorProxy.shape[tensorProxy.shape.Length - 1]; - foreach (int agentId in actionIds) + for (var i = 0; i < actionIds.Count; i++) { + var agentId = actionIds[i]; List memory; if (!m_Memories.TryGetValue(agentId, out memory) || memory.Count < memorySize * m_MemoriesCount) diff --git a/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs b/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs index 40853ec8ea..25c65cb87b 100644 --- a/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs +++ b/com.unity.ml-agents/Runtime/Inference/GeneratorImpl.cs @@ -21,7 +21,7 @@ public BiDimensionalOutputGenerator(ITensorAllocator allocator) m_Allocator = allocator; } - public void Generate(TensorProxy tensorProxy, int batchSize, IEnumerable infos) + public void Generate(TensorProxy tensorProxy, int batchSize, IList infos) { TensorUtils.ResizeTensor(tensorProxy, batchSize, m_Allocator); } @@ -40,7 +40,7 @@ public BatchSizeGenerator(ITensorAllocator allocator) m_Allocator = allocator; } - public void Generate(TensorProxy tensorProxy, int batchSize, IEnumerable infos) + public void Generate(TensorProxy tensorProxy, int batchSize, IList infos) { tensorProxy.data?.Dispose(); tensorProxy.data = m_Allocator.Alloc(new TensorShape(1, 1)); @@ -63,7 +63,7 @@ public SequenceLengthGenerator(ITensorAllocator allocator) m_Allocator = allocator; } - public void Generate(TensorProxy tensorProxy, int batchSize, IEnumerable infos) + public void Generate(TensorProxy tensorProxy, int batchSize, IList infos) { tensorProxy.shape = new long[0]; tensorProxy.data?.Dispose(); @@ -92,14 +92,15 @@ public RecurrentInputGenerator( } public void Generate( - TensorProxy tensorProxy, int batchSize, IEnumerable infos) + TensorProxy tensorProxy, int batchSize, IList infos) { TensorUtils.ResizeTensor(tensorProxy, batchSize, m_Allocator); var memorySize = tensorProxy.shape[tensorProxy.shape.Length - 1]; var agentIndex = 0; - foreach (var infoSensorPair in infos) + for (var infoIndex = 0; infoIndex < infos.Count; infoIndex++) { + var infoSensorPair = infos[infoIndex]; var info = infoSensorPair.agentInfo; List memory; @@ -147,14 +148,15 @@ public BarracudaRecurrentInputGenerator( m_Memories = memories; } - public void Generate(TensorProxy tensorProxy, int batchSize, IEnumerable infos) + public void Generate(TensorProxy tensorProxy, int batchSize, IList infos) { TensorUtils.ResizeTensor(tensorProxy, batchSize, m_Allocator); var memorySize = (int)tensorProxy.shape[tensorProxy.shape.Length - 1]; var agentIndex = 0; - foreach (var infoSensorPair in infos) + for (var infoIndex = 0; infoIndex < infos.Count; infoIndex++) { + var infoSensorPair = infos[infoIndex]; var info = infoSensorPair.agentInfo; var offset = memorySize * m_MemoryIndex; List memory; @@ -200,14 +202,15 @@ public PreviousActionInputGenerator(ITensorAllocator allocator) m_Allocator = allocator; } - public void Generate(TensorProxy tensorProxy, int batchSize, IEnumerable infos) + public void Generate(TensorProxy tensorProxy, int batchSize, IList infos) { TensorUtils.ResizeTensor(tensorProxy, batchSize, m_Allocator); var actionSize = tensorProxy.shape[tensorProxy.shape.Length - 1]; var agentIndex = 0; - foreach (var infoSensorPair in infos) + for (var infoIndex = 0; infoIndex < infos.Count; infoIndex++) { + var infoSensorPair = infos[infoIndex]; var info = infoSensorPair.agentInfo; var pastAction = info.storedActions.DiscreteActions; if (!pastAction.IsEmpty()) @@ -238,14 +241,15 @@ public ActionMaskInputGenerator(ITensorAllocator allocator) m_Allocator = allocator; } - public void Generate(TensorProxy tensorProxy, int batchSize, IEnumerable infos) + public void Generate(TensorProxy tensorProxy, int batchSize, IList infos) { TensorUtils.ResizeTensor(tensorProxy, batchSize, m_Allocator); var maskSize = tensorProxy.shape[tensorProxy.shape.Length - 1]; var agentIndex = 0; - foreach (var infoSensorPair in infos) + for (var infoIndex = 0; infoIndex < infos.Count; infoIndex++) { + var infoSensorPair = infos[infoIndex]; var agentInfo = infoSensorPair.agentInfo; var maskList = agentInfo.discreteActionMasks; for (var j = 0; j < maskSize; j++) @@ -274,7 +278,7 @@ public RandomNormalInputGenerator(int seed, ITensorAllocator allocator) m_Allocator = allocator; } - public void Generate(TensorProxy tensorProxy, int batchSize, IEnumerable infos) + public void Generate(TensorProxy tensorProxy, int batchSize, IList infos) { TensorUtils.ResizeTensor(tensorProxy, batchSize, m_Allocator); TensorUtils.FillTensorWithRandomNormal(tensorProxy, m_RandomNormal); @@ -303,12 +307,13 @@ public void AddSensorIndex(int sensorIndex) m_SensorIndices.Add(sensorIndex); } - public void Generate(TensorProxy tensorProxy, int batchSize, IEnumerable infos) + public void Generate(TensorProxy tensorProxy, int batchSize, IList infos) { TensorUtils.ResizeTensor(tensorProxy, batchSize, m_Allocator); var agentIndex = 0; - foreach (var info in infos) + for (var infoIndex = 0; infoIndex < infos.Count; infoIndex++) { + var info = infos[infoIndex]; if (info.agentInfo.done) { // If the agent is done, we might have a stale reference to the sensors @@ -320,8 +325,9 @@ public void Generate(TensorProxy tensorProxy, int batchSize, IEnumerable m_InferenceInputs; - IReadOnlyList m_InferenceOutputs; + List m_InferenceOutputs; + Dictionary m_InputsByName; Dictionary> m_Memories = new Dictionary>(); SensorShapeValidator m_SensorShapeValidator = new SensorShapeValidator(); @@ -56,6 +58,7 @@ public ModelRunner( { Model barracudaModel; m_Model = model; + m_ModelName = model.name; m_InferenceDevice = inferenceDevice; m_TensorAllocator = new TensorCachingAllocator(); if (model != null) @@ -84,6 +87,8 @@ public ModelRunner( seed, m_TensorAllocator, m_Memories, barracudaModel); m_TensorApplier = new TensorApplier( actionSpec, seed, m_TensorAllocator, m_Memories, barracudaModel); + m_InputsByName = new Dictionary(); + m_InferenceOutputs = new List(); } public InferenceDevice InferenceDevice @@ -96,15 +101,14 @@ public NNModel Model get { return m_Model; } } - static Dictionary PrepareBarracudaInputs(IEnumerable infInputs) + void PrepareBarracudaInputs(IReadOnlyList infInputs) { - var inputs = new Dictionary(); - foreach (var inp in infInputs) + m_InputsByName.Clear(); + for (var i = 0; i < infInputs.Count; i++) { - inputs[inp.name] = inp.data; + var inp = infInputs[i]; + m_InputsByName[inp.name] = inp.data; } - - return inputs; } public void Dispose() @@ -114,16 +118,14 @@ public void Dispose() m_TensorAllocator?.Reset(false); } - List FetchBarracudaOutputs(string[] names) + void FetchBarracudaOutputs(string[] names) { - var outputs = new List(); + m_InferenceOutputs.Clear(); foreach (var n in names) { var output = m_Engine.PeekOutput(n); - outputs.Add(TensorUtils.TensorProxyFromBarracuda(output, n)); + m_InferenceOutputs.Add(TensorUtils.TensorProxyFromBarracuda(output, n)); } - - return outputs; } public void PutObservations(AgentInfo info, List sensors) @@ -169,31 +171,33 @@ public void DecideBatch() } Profiler.BeginSample("ModelRunner.DecideAction"); + Profiler.BeginSample(m_ModelName); - Profiler.BeginSample($"MLAgents.{m_Model.name}.GenerateTensors"); + Profiler.BeginSample($"GenerateTensors"); // Prepare the input tensors to be feed into the engine m_TensorGenerator.GenerateTensors(m_InferenceInputs, currentBatchSize, m_Infos); Profiler.EndSample(); - Profiler.BeginSample($"MLAgents.{m_Model.name}.PrepareBarracudaInputs"); - var inputs = PrepareBarracudaInputs(m_InferenceInputs); + Profiler.BeginSample($"PrepareBarracudaInputs"); + PrepareBarracudaInputs(m_InferenceInputs); Profiler.EndSample(); // Execute the Model - Profiler.BeginSample($"MLAgents.{m_Model.name}.ExecuteGraph"); - m_Engine.Execute(inputs); + Profiler.BeginSample($"ExecuteGraph"); + m_Engine.Execute(m_InputsByName); Profiler.EndSample(); - Profiler.BeginSample($"MLAgents.{m_Model.name}.FetchBarracudaOutputs"); - m_InferenceOutputs = FetchBarracudaOutputs(m_OutputNames); + Profiler.BeginSample($"FetchBarracudaOutputs"); + FetchBarracudaOutputs(m_OutputNames); Profiler.EndSample(); - Profiler.BeginSample($"MLAgents.{m_Model.name}.ApplyTensors"); + Profiler.BeginSample($"ApplyTensors"); // Update the outputs m_TensorApplier.ApplyTensors(m_InferenceOutputs, m_OrderedAgentsRequestingDecisions, m_LastActionsReceived); Profiler.EndSample(); - Profiler.EndSample(); + Profiler.EndSample(); // end name + Profiler.EndSample(); // end ModelRunner.DecideAction m_Infos.Clear(); diff --git a/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs b/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs index 72b2eca84d..d3d2d393f7 100644 --- a/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs +++ b/com.unity.ml-agents/Runtime/Inference/TensorApplier.cs @@ -31,7 +31,7 @@ public interface IApplier /// /// List of Agents Ids that will be updated using the tensor's data /// Dictionary of AgentId to Actions to be updated - void Apply(TensorProxy tensorProxy, IEnumerable actionIds, Dictionary lastActions); + void Apply(TensorProxy tensorProxy, IList actionIds, Dictionary lastActions); } readonly Dictionary m_Dict = new Dictionary(); @@ -90,10 +90,11 @@ public TensorApplier( /// One of the tensor does not have an /// associated applier. public void ApplyTensors( - IEnumerable tensors, IEnumerable actionIds, Dictionary lastActions) + IReadOnlyList tensors, IList actionIds, Dictionary lastActions) { - foreach (var tensor in tensors) + for (var tensorIndex = 0; tensorIndex < tensors.Count; tensorIndex++) { + var tensor = tensors[tensorIndex]; if (!m_Dict.ContainsKey(tensor.name)) { throw new UnityAgentsException( diff --git a/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs b/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs index b8650b6c01..02194ca6b5 100644 --- a/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs +++ b/com.unity.ml-agents/Runtime/Inference/TensorGenerator.cs @@ -31,7 +31,7 @@ public interface IGenerator /// the tensor's data. /// void Generate( - TensorProxy tensorProxy, int batchSize, IEnumerable infos); + TensorProxy tensorProxy, int batchSize, IList infos); } readonly Dictionary m_Dict = new Dictionary(); @@ -149,10 +149,11 @@ public void InitializeObservations(List sensors, ITensorAllocator alloc /// One of the tensor does not have an /// associated generator. public void GenerateTensors( - IEnumerable tensors, int currentBatchSize, IEnumerable infos) + IReadOnlyList tensors, int currentBatchSize, IList infos) { - foreach (var tensor in tensors) + for (var tensorIndex = 0; tensorIndex < tensors.Count; tensorIndex++) { + var tensor = tensors[tensorIndex]; if (!m_Dict.ContainsKey(tensor.name)) { throw new UnityAgentsException( diff --git a/com.unity.ml-agents/Runtime/Sensors/ObservationWriter.cs b/com.unity.ml-agents/Runtime/Sensors/ObservationWriter.cs index c27dc79a94..bbdb4e4622 100644 --- a/com.unity.ml-agents/Runtime/Sensors/ObservationWriter.cs +++ b/com.unity.ml-agents/Runtime/Sensors/ObservationWriter.cs @@ -64,7 +64,7 @@ internal void SetTarget(TensorProxy tensorProxy, int batchIndex, int channelOffs } /// - /// 1D write access at a specified index. Use AddRange if possible instead. + /// 1D write access at a specified index. Use AddList if possible instead. /// /// Index to write to. public float this[int index] @@ -122,6 +122,7 @@ public float this[int index] /// /// /// Optional write offset. + [Obsolete("Use AddList() for better performance")] public void AddRange(IEnumerable data, int writeOffset = 0) { if (m_Data != null) @@ -144,6 +145,27 @@ public void AddRange(IEnumerable data, int writeOffset = 0) } } + public void AddList(IList data, int writeOffset = 0) + { + if (m_Data != null) + { + for (var index = 0; index < data.Count; index++) + { + var val = data[index]; + m_Data[index + m_Offset + writeOffset] = val; + + } + } + else + { + for (var index = 0; index < data.Count; index++) + { + var val = data[index]; + m_Proxy.data[m_Batch, index + m_Offset + writeOffset] = val; + } + } + } + /// /// Write the Vector3 components. /// diff --git a/com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensor.cs b/com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensor.cs index 1f84b7c520..2b0b62cfca 100644 --- a/com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensor.cs +++ b/com.unity.ml-agents/Runtime/Sensors/RayPerceptionSensor.cs @@ -329,7 +329,7 @@ public int Write(ObservationWriter writer) rayOutput.ToFloatArray(numDetectableTags, rayIndex, m_Observations); } // Finally, add the observations to the ObservationWriter - writer.AddRange(m_Observations); + writer.AddList(m_Observations); } return m_Observations.Length; } diff --git a/com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs b/com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs index b9d55f6060..80c914c3ea 100644 --- a/com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs +++ b/com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs @@ -112,7 +112,7 @@ public int Write(ObservationWriter writer) for (var i = 0; i < m_NumStackedObservations; i++) { var obsIndex = (m_CurrentIndex + 1 + i) % m_NumStackedObservations; - writer.AddRange(m_StackedObservations[obsIndex], numWritten); + writer.AddList(m_StackedObservations[obsIndex], numWritten); numWritten += m_UnstackedObservationSize; } } diff --git a/com.unity.ml-agents/Runtime/Sensors/VectorSensor.cs b/com.unity.ml-agents/Runtime/Sensors/VectorSensor.cs index 6c60c5c81c..d5e028c01d 100644 --- a/com.unity.ml-agents/Runtime/Sensors/VectorSensor.cs +++ b/com.unity.ml-agents/Runtime/Sensors/VectorSensor.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Collections.ObjectModel; using UnityEngine; @@ -57,7 +58,7 @@ public int Write(ObservationWriter writer) m_Observations.Add(0); } } - writer.AddRange(m_Observations); + writer.AddList(m_Observations); return expectedObservations; } @@ -164,6 +165,7 @@ public void AddObservation(Vector2 observation) /// Adds a collection of float observations to the vector observations of the agent. /// /// Observation. + [Obsolete("Use AddObservation(IList) for better performance.")] public void AddObservation(IEnumerable observation) { foreach (var f in observation) @@ -172,6 +174,18 @@ public void AddObservation(IEnumerable observation) } } + /// + /// Adds a list or array of float observations to the vector observations of the agent. + /// + /// Observation. + public void AddObservation(IList observation) + { + for (var i = 0; i < observation.Count; i++) + { + AddFloatObs(observation[i]); + } + } + /// /// Adds a quaternion observation to the vector observations of the agent. /// diff --git a/com.unity.ml-agents/Tests/Editor/Sensor/ObservationWriterTests.cs b/com.unity.ml-agents/Tests/Editor/Sensor/ObservationWriterTests.cs index 859d26d6a8..813c716a88 100644 --- a/com.unity.ml-agents/Tests/Editor/Sensor/ObservationWriterTests.cs +++ b/com.unity.ml-agents/Tests/Editor/Sensor/ObservationWriterTests.cs @@ -26,14 +26,14 @@ public void TestWritesToIList() writer[0] = 3f; Assert.AreEqual(new[] { 1f, 3f, 2f }, buffer); - // AddRange + // AddList writer.SetTarget(buffer, shape, 0); - writer.AddRange(new[] { 4f, 5f }); + writer.AddList(new[] { 4f, 5f }); Assert.AreEqual(new[] { 4f, 5f, 2f }, buffer); - // AddRange with offset + // AddList with offset writer.SetTarget(buffer, shape, 1); - writer.AddRange(new[] { 6f, 7f }); + writer.AddList(new[] { 6f, 7f }); Assert.AreEqual(new[] { 4f, 6f, 7f }, buffer); } @@ -60,7 +60,7 @@ public void TestWritesToTensor() Assert.AreEqual(2f, t.data[1, 1]); Assert.AreEqual(3f, t.data[1, 2]); - // AddRange + // AddList t = new TensorProxy { valueType = TensorProxy.TensorType.FloatingPoint, @@ -68,7 +68,7 @@ public void TestWritesToTensor() }; writer.SetTarget(t, 1, 1); - writer.AddRange(new[] { -1f, -2f }); + writer.AddList(new[] { -1f, -2f }); Assert.AreEqual(0f, t.data[0, 0]); Assert.AreEqual(0f, t.data[0, 1]); Assert.AreEqual(0f, t.data[0, 2]); diff --git a/docs/Migrating.md b/docs/Migrating.md index dd0d6f77e2..0eadcb9f06 100644 --- a/docs/Migrating.md +++ b/docs/Migrating.md @@ -17,7 +17,10 @@ double-check that the versions are in the same. The versions can be found in ## Migrating to Release 13 ### Implementing IHeuristic in your IActuator implementations - If you have any custom actuators, you can now implement the `IHeuristicProvider` interface to have your actuator -handle the generation of actions when an Agent is running in heuristic mode. + handle the generation of actions when an Agent is running in heuristic mode. +- `VectorSensor.AddObservation(IEnumerable)` is deprecated. Use `VectorSensor.AddObservation(IList)` + instead. +- `ObservationWriter.AddRange()` is deprecated. Use `ObservationWriter.AddList()` instead. # Migrating