Skip to content

Adding more tests for v2 models #5138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 17, 2021
Merged

Adding more tests for v2 models #5138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
64ba9c1
Adding more tests for v2 models
vincentpierre Mar 16, 2021
b66161a
formatting
vincentpierre Mar 16, 2021
5c02974
fixing formatting
vincentpierre Mar 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ namespace Unity.MLAgents.Tests.Analytics
[TestFixture]
public class InferenceAnalyticsTests
{
const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action.onnx";
const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action_v1_0.onnx";
NNModel continuousONNXModel;
Test3DSensorComponent sensor_21_20_3;
Test3DSensorComponent sensor_20_22_3;
Expand Down
2 changes: 1 addition & 1 deletion com.unity.ml-agents/Tests/Editor/BehaviorParameterTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ namespace Unity.MLAgents.Tests
[TestFixture]
public class BehaviorParameterTests : IHeuristicProvider
{
const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action.onnx";
const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action_v1_0.onnx";
public void Heuristic(in ActionBuffers actionsOut)
{
// No-op
Expand Down
8 changes: 8 additions & 0 deletions com.unity.ml-agents/Tests/Editor/Inference.meta
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 ...Editor/DiscreteActionOutputApplierTest.cs → ...erence/DiscreteActionOutputApplierTest.cs
View file
Open in desktop
File renamed without changes.
0 ...r/DiscreteActionOutputApplierTest.cs.meta → ...e/DiscreteActionOutputApplierTest.cs.meta
View file
Open in desktop
File renamed without changes.
0 ...EditModeTestInternalBrainTensorApplier.cs → ...EditModeTestInternalBrainTensorApplier.cs
View file
Open in desktop
File renamed without changes.
0 ...odeTestInternalBrainTensorApplier.cs.meta → ...odeTestInternalBrainTensorApplier.cs.meta
View file
Open in desktop
File renamed without changes.
0 ...itModeTestInternalBrainTensorGenerator.cs → ...itModeTestInternalBrainTensorGenerator.cs
View file
Open in desktop
File renamed without changes.
0 ...eTestInternalBrainTensorGenerator.cs.meta → ...eTestInternalBrainTensorGenerator.cs.meta
View file
Open in desktop
File renamed without changes.
12 changes: 7 additions & 5 deletions ...ml-agents/Tests/Editor/ModelRunnerTest.cs → ...Tests/Editor/Inference/ModelRunnerTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@ namespace Unity.MLAgents.Tests
[TestFixture]
public class ModelRunnerTest
{
const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action.onnx";
const string k_discreteONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/discrete1vis0vec_2_3action_recurr.onnx";
const string k_hybridONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/hybrid0vis53vec_3c_2daction.onnx";
const string k_continuousNNPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action_deprecated.nn";
const string k_discreteNNPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/discrete1vis0vec_2_3action_recurr_deprecated.nn";
const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action_v1_0.onnx";
const string k_discreteONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/discrete1vis0vec_2_3action_recurr_v1_0.onnx";
const string k_hybridONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/hybrid0vis53vec_3c_2daction_v1_0.onnx";
const string k_continuousNNPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action_deprecated_v1_0.nn";
const string k_discreteNNPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/discrete1vis0vec_2_3action_recurr_deprecated_v1_0.nn";

NNModel continuousONNXModel;
NNModel discreteONNXModel;
NNModel hybridONNXModel;
Expand All @@ -25,6 +26,7 @@ public class ModelRunnerTest
Test3DSensorComponent sensor_21_20_3;
Test3DSensorComponent sensor_20_22_3;


ActionSpec GetContinuous2vis8vec2actionActionSpec()
{
return ActionSpec.MakeContinuous(2);
Expand Down
0 ...ents/Tests/Editor/ModelRunnerTest.cs.meta → .../Editor/Inference/ModelRunnerTest.cs.meta
View file
Open in desktop
File renamed without changes.
62 changes: 57 additions & 5 deletions ...gents/Tests/Editor/ParameterLoaderTest.cs → ...s/Editor/Inference/ParameterLoaderTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,21 +92,28 @@ public DimensionProperty[] GetDimensionProperties()
[TestFixture]
public class ParameterLoaderTest
{
const string k_discrete_ONNX_v2 = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/discrete_rank2_vector_v2_0.onnx";

// ONNX model with continuous/discrete action output (support hybrid action)
const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action.onnx";
const string k_discreteONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/discrete1vis0vec_2_3action_recurr.onnx";
const string k_hybridONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/hybrid0vis53vec_3c_2daction.onnx";
const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action_v1_0.onnx";
const string k_discreteONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/discrete1vis0vec_2_3action_recurr_v1_0.onnx";
const string k_hybridONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/hybrid0vis53vec_3c_2daction_v1_0.onnx";
// NN model with single action output (deprecated, does not support hybrid action).
// Same BrainParameters settings as the corresponding ONNX model.
const string k_continuousNNPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action_deprecated.nn";
const string k_discreteNNPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/discrete1vis0vec_2_3action_recurr_deprecated.nn";
const string k_continuousNNPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action_deprecated_v1_0.nn";
const string k_discreteNNPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/discrete1vis0vec_2_3action_recurr_deprecated_v1_0.nn";

NNModel rank2ONNXModel;
NNModel continuousONNXModel;
NNModel discreteONNXModel;
NNModel hybridONNXModel;
NNModel continuousNNModel;
NNModel discreteNNModel;
Test3DSensorComponent sensor_21_20_3;
Test3DSensorComponent sensor_20_22_3;
BufferSensor sensor_23_20;
VectorSensor sensor_8;
VectorSensor sensor_10;

BrainParameters GetContinuous2vis8vec2actionBrainParameters()
{
Expand Down Expand Up @@ -135,6 +142,15 @@ BrainParameters GetHybridBrainParameters()
return validBrainParameters;
}

BrainParameters GetRank2BrainParameters()
{
var validBrainParameters = new BrainParameters();
validBrainParameters.VectorObservationSize = 4;
validBrainParameters.NumStackedVectorObservations = 2;
validBrainParameters.ActionSpec = ActionSpec.MakeDiscrete(3, 3, 3);
return validBrainParameters;
}

[SetUp]
public void SetUp()
{
Expand All @@ -143,11 +159,15 @@ public void SetUp()
hybridONNXModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_hybridONNXPath, typeof(NNModel));
continuousNNModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_continuousNNPath, typeof(NNModel));
discreteNNModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_discreteNNPath, typeof(NNModel));
rank2ONNXModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_discrete_ONNX_v2, typeof(NNModel));
var go = new GameObject("SensorA");
sensor_21_20_3 = go.AddComponent<Test3DSensorComponent>();
sensor_21_20_3.Sensor = new Test3DSensor("SensorA", 21, 20, 3);
sensor_20_22_3 = go.AddComponent<Test3DSensorComponent>();
sensor_20_22_3.Sensor = new Test3DSensor("SensorA", 20, 22, 3);
sensor_23_20 = new BufferSensor(20, 23, "BufferSensor");
sensor_8 = new VectorSensor(8, "VectorSensor8");
sensor_10 = new VectorSensor(10, "VectorSensor10");
}

[Test]
Expand All @@ -158,6 +178,7 @@ public void TestModelExist()
Assert.IsNotNull(hybridONNXModel);
Assert.IsNotNull(continuousNNModel);
Assert.IsNotNull(discreteNNModel);
Assert.IsNotNull(rank2ONNXModel);
}

[TestCase(true)]
Expand Down Expand Up @@ -239,6 +260,37 @@ public void TestGetOutputTensorsHybrid()
Assert.AreEqual(0, model.GetOutputNames().Count());
}

[Test]
public void TestCheckModelRank2()
{
var model = ModelLoader.Load(rank2ONNXModel);
var validBrainParameters = GetRank2BrainParameters();

var errors = BarracudaModelParamLoader.CheckModel(
model, validBrainParameters,
new ISensor[] { sensor_23_20, sensor_10, sensor_8 }, new ActuatorComponent[0]
);
Assert.AreEqual(0, errors.Count()); // There should not be any errors

errors = BarracudaModelParamLoader.CheckModel(
model, validBrainParameters,
new ISensor[] { sensor_23_20, sensor_10 }, new ActuatorComponent[0]
);
Assert.AreNotEqual(0, errors.Count()); // Wrong number of sensors

errors = BarracudaModelParamLoader.CheckModel(
model, validBrainParameters,
new ISensor[] { new BufferSensor(20, 40, "BufferSensor"), sensor_10, sensor_8 }, new ActuatorComponent[0]
);
Assert.AreNotEqual(0, errors.Count()); // Wrong buffer sensor size

errors = BarracudaModelParamLoader.CheckModel(
model, validBrainParameters,
new ISensor[] { sensor_23_20, sensor_10, sensor_10 }, new ActuatorComponent[0]
);
Assert.AreNotEqual(0, errors.Count()); // Wrong vector sensor size
}

[TestCase(true)]
[TestCase(false)]
public void TestCheckModelValidContinuous(bool useDeprecatedNNModel)
Expand Down
0 .../Tests/Editor/ParameterLoaderTest.cs.meta → ...tor/Inference/ParameterLoaderTest.cs.meta
View file
Open in desktop
File renamed without changes.
0 ...ml-agents/Tests/Editor/TensorUtilsTest.cs → ...Tests/Editor/Inference/TensorUtilsTest.cs
View file
Open in desktop
File renamed without changes.
0 ...ents/Tests/Editor/TensorUtilsTest.cs.meta → .../Editor/Inference/TensorUtilsTest.cs.meta
View file
Open in desktop
File renamed without changes.
0 ...s/continuous2vis8vec2action_deprecated.nn → ...tinuous2vis8vec2action_deprecated_v1_0.nn
View file
Open in desktop
File renamed without changes.
0 ...tinuous2vis8vec2action_deprecated.nn.meta → ...us2vis8vec2action_deprecated_v1_0.nn.meta
View file
Open in desktop
File renamed without changes.
0 ...TestModels/continuous2vis8vec2action.onnx → ...odels/continuous2vis8vec2action_v1_0.onnx
View file
Open in desktop
File renamed without changes.
0 ...odels/continuous2vis8vec2action.onnx.meta → .../continuous2vis8vec2action_v1_0.onnx.meta
View file
Open in desktop
File renamed without changes.
0 ...te1vis0vec_2_3action_recurr_deprecated.nn → ...s0vec_2_3action_recurr_deprecated_v1_0.nn
View file
Open in desktop
File renamed without changes.
0 ...s0vec_2_3action_recurr_deprecated.nn.meta → ..._2_3action_recurr_deprecated_v1_0.nn.meta
View file
Open in desktop
File renamed without changes.
0 ...ls/discrete1vis0vec_2_3action_recurr.onnx → ...screte1vis0vec_2_3action_recurr_v1_0.onnx
View file
Open in desktop
File renamed without changes.
0 ...screte1vis0vec_2_3action_recurr.onnx.meta → ...e1vis0vec_2_3action_recurr_v1_0.onnx.meta
View file
Open in desktop
File renamed without changes.
Binary file added com.unity.ml-agents/Tests/Editor/TestModels/discrete_rank2_vector_v2_0.onnx
View file
Open in desktop
Binary file not shown.
15 changes: 15 additions & 0 deletions com.unity.ml-agents/Tests/Editor/TestModels/discrete_rank2_vector_v2_0.onnx.meta
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 ...stModels/hybrid0vis53vec_3c_2daction.onnx → ...els/hybrid0vis53vec_3c_2daction_v1_0.onnx
View file
Open in desktop
File renamed without changes.
0 ...els/hybrid0vis53vec_3c_2daction.onnx.meta → ...ybrid0vis53vec_3c_2daction_v1_0.onnx.meta
View file
Open in desktop
File renamed without changes.