TestResultCoordinator: Plumb down test context into reports (#5902)

***Please replace this line with your PR description and read PR checklist below*** Currently longhaul pass results don't indicate the current release criteria. I am updating the tolerances for our tests in order to make it easier on the release manager to know what is releasable vs what requires further investigation. The first step in this direction is to plumb down two pieces of information into the test reports: 1. Whether or not the broker is enabled (`MqttBrokerEnabled` flag) 2. Whether topology is single node (`Topology` enum). This used to come from the deployment file, but there was a lot of duplication there and it is simpler if it comes from an environment variable. Tests running to confirm these changes work: 1. Longhaul: https://msazure.visualstudio.com/One/_build/results?buildId=49707570&view=results 2. Nested Longhaul: https://msazure.visualstudio.com/One/_build/results?buildId=49707599&view=results 3. Connectivity: https://msazure.visualstudio.com/One/_build/results?buildId=49707606&view=results 4. Nested Connectivity: https://msazure.visualstudio.com/One/_build/results?buildId=49707634&view=results 5. End to End: https://dev.azure.com/msazure/One/_build/results?buildId=49707665&view=results 6. Nested End-to-End : https://msazure.visualstudio.com/One/_build/results?buildId=49712331&view=results ## Azure IoT Edge PR checklist:
Azure · Dec 8, 2021 · 6b9a85f · 6b9a85f
1 parent d9fff23
commit 6b9a85f
Show file tree

Hide file tree

Showing 28 changed files with 228 additions and 39 deletions.
diff --git a/builds/e2e/connectivity.yaml b/builds/e2e/connectivity.yaml
@@ -218,6 +218,7 @@ jobs:
  customEdgeHub.image: '$(customEdgeHub.image)'
  test.runtimeLogLevel: '$(test.runtimeLogLevel)'
  testInfo.testName: '$(testInfo.testName)'
+ topology: 'SingleNode'
 
 
 ################################################################################
@@ -398,4 +399,5 @@ jobs:
  customEdgeHub.image: '$(customEdgeHub.image)'
  test.runtimeLogLevel: '$(test.runtimeLogLevel)'
  connectivity.nested: 'false'
- testInfo.testName: '$(testInfo.testName)'
+ testInfo.testName: '$(testInfo.testName)'
+ topology: 'SingleNode'
diff --git a/builds/e2e/nested-connectivity.yaml b/builds/e2e/nested-connectivity.yaml
@@ -176,6 +176,7 @@ stages:
  aziotis.package.filter: '$(aziotis.package.filter)'
  connectivity.nested: 'true'
  testInfo.testName: '$(testInfo.testName)'
+ topology: 'Nested'
 
 
 - stage: Cleanup 

diff --git a/builds/e2e/templates/connectivity-deploy.yaml b/builds/e2e/templates/connectivity-deploy.yaml
@@ -30,6 +30,7 @@ parameters:
  customEdgeAgent.image: ''
  customEdgeHub.image: ''
  test.runtimeLogLevel: ''
+ topology: ''
  connectivity.nested: 'false'
 
 steps:
@@ -143,6 +144,7 @@ steps:
  -testRuntimeLogLevel "${{ parameters['test.runtimeLogLevel'] }}" \
  -testInfo "$testInfo" \
  -repoPath "${{ parameters['build.repo.path'] }}" \
+ -topology "${{ parameters['topology'] }}" \
  -waitForTestComplete \
  -cleanAll
 

diff --git a/e2e_deployment_files/connectivity_deployment.template.json b/e2e_deployment_files/connectivity_deployment.template.json
@@ -33,7 +33,7 @@
  "ExperimentalFeatures__EnableUploadLogs": {
  "value": "true"
  },
- "RuntimeLogLevel" : {
+ "RuntimeLogLevel": {
  "value": "debug"
  }
  },
@@ -399,6 +399,9 @@
  },
  "TEST_INFO": {
  "value": "<TestInfo>"
+ },
+ "topology": {
+ "value": "<Topology>"
  }
  },
  "settings": {

diff --git a/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_connectivity_mqtt.template.json b/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_connectivity_mqtt.template.json
@@ -387,6 +387,12 @@
  },
  "TEST_INFO": {
  "value": "<TestInfo>"
+ },
+ "topology": {
+ "value": "Nested"
+ },
+ "mqttBrokerEnabled": {
+ "value": "true"
  }
  },
  "settings": {
@@ -726,32 +732,28 @@
  "TwinTestPropertyType": "Desired",
  "ExpectedSource": "twinTester1.desiredUpdated",
  "ActualSource": "twinTester2.desiredReceived",
- "TestDescription": "twin | desired property | amqp",
- "Topology": "Nested"
+ "TestDescription": "twin | desired property | amqp"
  },
  "reportMetadata6": {
  "TestReportType": "TwinCountingReport",
  "TwinTestPropertyType": "Reported",
  "ExpectedSource": "twinTester2.reportedReceived",
  "ActualSource": "twinTester2.reportedUpdated",
- "TestDescription": "twin | reported property | amqp",
- "Topology": "Nested"
+ "TestDescription": "twin | reported property | amqp"
  },
  "reportMetadata7": {
  "TestReportType": "TwinCountingReport",
  "TwinTestPropertyType": "Desired",
  "ExpectedSource": "twinTester3.desiredUpdated",
  "ActualSource": "twinTester4.desiredReceived",
- "TestDescription": "twin | desired property | mqtt",
- "Topology": "Nested"
+ "TestDescription": "twin | desired property | mqtt"
  },
  "reportMetadata8": {
  "TestReportType": "TwinCountingReport",
  "TwinTestPropertyType": "Reported",
  "ExpectedSource": "twinTester4.reportedReceived",
  "ActualSource": "twinTester4.reportedUpdated",
- "TestDescription": "twin | reported property | mqtt",
- "Topology": "Nested"
+ "TestDescription": "twin | reported property | mqtt"
  },
  "reportMetadata9": {
  "TestReportType": "DeploymentTestReport",
@@ -764,23 +766,20 @@
  "SenderSource": "directMethodSender1.send",
  "ReceiverSource": "directMethodReceiver1.receive",
  "TolerancePeriod": "00:01:00",
- "TestDescription": "direct method | cloud | amqp",
- "Topology": "Nested"
+ "TestDescription": "direct method | cloud | amqp"
  },
  "reportMetadata11": {
  "TestReportType": "DirectMethodConnectivityReport",
  "SenderSource": "directMethodSender2.send",
  "ReceiverSource": "directMethodReceiver2.receive",
  "TolerancePeriod": "00:01:00",
- "TestDescription": "direct method | cloud | mqtt",
- "Topology": "Nested"
+ "TestDescription": "direct method | cloud | mqtt"
  },
  "reportMetadata12": {
  "TestReportType": "DirectMethodConnectivityReport",
  "SenderSource": "directMethodSender3.send",
  "TolerancePeriod": "00:01:00",
- "TestDescription": "edge agent ping",
- "Topology": "Nested"
+ "TestDescription": "edge agent ping"
  },
  "reportMetadata15": {
  "TestReportType": "NetworkControllerReport",

diff --git a/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_amqp.template.json b/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_amqp.template.json
@@ -399,6 +399,9 @@
  },
  "testMode": {
  "value": "<TestMode>"
+ },
+ "topology": {
+ "value": "Nested"
  }
  },
  "settings": {

diff --git a/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_mqtt.template.json b/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_mqtt.template.json
@@ -408,6 +408,12 @@
  },
  "testMode": {
  "value": "<TestMode>"
+ },
+ "topology": {
+ "value": "Nested"
+ },
+ "mqttBrokerEnabled": {
+ "value": "true"
  }
  },
  "settings": {

diff --git a/scripts/linux/trcE2ETest.sh b/scripts/linux/trcE2ETest.sh
@@ -50,6 +50,7 @@ function usage() {
  echo ' -restartIntervalInMins Value for long haul specifying how often a random module will restart. If specified, then "desiredModulesToRestartCSV" must be specified as well.'
  echo ' -sendReportFrequency Value for long haul specifying how often TRC will send reports to LogAnalytics.'
  echo " -testMode Test mode for TestResultCoordinator to start up with correct settings. Value is either 'LongHaul' or 'Connectivity'."
+ echo " -topology Configuration telling the TRC which topology tests are running in."
  echo " -repoPath Path of the checked-out iotedge repository for getting the deployment file."
  echo " -clientModuleTransportType Value for contrained long haul specifying transport type for all client modules."
  echo " -trackingId Tracking id used to tag test events. Needed if running nested tests and test events are sent to TRC from L4 node. Otherwise generated."
@@ -211,6 +212,7 @@ function prepare_test_from_artifacts() {
  sed -i -e "s@<NetworkController.RunsCount0>@${NETWORK_CONTROLLER_FREQUENCIES[2]}@g" "$deployment_working_file"
 
  sed -i -e "s@<TestMode>@$TEST_MODE@g" "$deployment_working_file"
+ sed -i -e "s@<Topology>@$TOPOLOGY@g" "$deployment_working_file"
 
  sed -i -e "s@<LogRotationMaxFile>@$log_rotation_max_file@g" "$deployment_working_file"
  sed -i -e "s@<LogRotationMaxFileEdgeHub>@$log_rotation_max_file_edgehub@g" "$deployment_working_file"
@@ -456,6 +458,9 @@ function process_args() {
  elif [ $saveNextArg -eq 47 ]; then
  TRACKING_ID="$arg"
  saveNextArg=0;
+ elif [ $saveNextArg -eq 48 ]; then
+ TOPOLOGY="$arg"
+ saveNextArg=0;
  else
  case "$arg" in
  '-h' | '--help' ) usage;;
@@ -506,6 +511,7 @@ function process_args() {
  '-repoPath' ) saveNextArg=45;;
  '-clientModuleTransportType' ) saveNextArg=46;;
  '-trackingId' ) saveNextArg=47;;
+ '-topology' ) saveNextArg=48;;
  '-waitForTestComplete' ) WAIT_FOR_TEST_COMPLETE=1;;
  '-cleanAll' ) CLEAN_ALL=1;;
 

diff --git a/test/Microsoft.Azure.Devices.Edge.Test/GenericMqtt.cs b/test/Microsoft.Azure.Devices.Edge.Test/GenericMqtt.cs
@@ -47,7 +47,7 @@ public async Task GenericMqttTelemetry()
 
  Action<EdgeConfigBuilder> addMqttBrokerConfig = MqttBrokerUtil.BuildAddBrokerToDeployment(false);
  Action<EdgeConfigBuilder> addNetworkControllerConfig = TestResultCoordinatorUtil.BuildAddNetworkControllerConfig(trackingId, networkControllerImage);
- Action<EdgeConfigBuilder> addTestResultCoordinatorConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, GenericMqttInitiatorModuleName, GenericMqttInitiatorModuleName);
+ Action<EdgeConfigBuilder> addTestResultCoordinatorConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, GenericMqttInitiatorModuleName, GenericMqttInitiatorModuleName, true);
  Action<EdgeConfigBuilder> addGenericMqttTesterConfig = this.BuildAddGenericMqttTesterConfig(trackingId, trcImage, genericMqttTesterImage);
  Action<EdgeConfigBuilder> config = addMqttBrokerConfig + addNetworkControllerConfig + addTestResultCoordinatorConfig + addGenericMqttTesterConfig;
  EdgeDeployment deployment = await this.runtime.DeployConfigurationAsync(config, token, Context.Current.NestedEdge);

diff --git a/test/Microsoft.Azure.Devices.Edge.Test/PriorityQueues.cs b/test/Microsoft.Azure.Devices.Edge.Test/PriorityQueues.cs
@@ -44,7 +44,7 @@ public async Task PriorityQueueModuleToModuleMessages()
  TestInfo testInfo = this.InitTestInfo(5, 1000, true);
 
  Action<EdgeConfigBuilder> addLoadGenConfig = this.BuildAddLoadGenConfig(trackingId, loadGenImage, testInfo, false);
- Action<EdgeConfigBuilder> addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, RelayerModuleName);
+ Action<EdgeConfigBuilder> addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, RelayerModuleName, false);
 
  EdgeDeployment deployment = await this.runtime.DeployConfigurationAsync(addLoadGenConfig + addTrcConfig, token, Context.Current.NestedEdge);
  PriorityQueueTestStatus loadGenTestStatus = await this.PollUntilFinishedAsync(LoadGenModuleName, token);
@@ -69,7 +69,7 @@ public async Task PriorityQueueModuleToHubMessages()
  var testResultReportingClient = new TestResultReportingClient { BaseUrl = "http://localhost:5001" };
 
  Action<EdgeConfigBuilder> addLoadGenConfig = this.BuildAddLoadGenConfig(trackingId, loadGenImage, testInfo, true);
- Action<EdgeConfigBuilder> addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, "hubtest");
+ Action<EdgeConfigBuilder> addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, "hubtest", false);
  Action<EdgeConfigBuilder> addNetworkControllerConfig = TestResultCoordinatorUtil.BuildAddNetworkControllerConfig(trackingId, networkControllerImage);
 
  EdgeDeployment deployment = await this.runtime.DeployConfigurationAsync(addLoadGenConfig + addTrcConfig + addNetworkControllerConfig, token, Context.Current.NestedEdge);
@@ -100,7 +100,7 @@ public async Task PriorityQueueTimeToLive()
  TestInfo testInfo = this.InitTestInfo(5, 20);
 
  Action<EdgeConfigBuilder> addLoadGenConfig = this.BuildAddLoadGenConfig(trackingId, loadGenImage, testInfo, false);
- Action<EdgeConfigBuilder> addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, RelayerModuleName);
+ Action<EdgeConfigBuilder> addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, RelayerModuleName, false);
 
  EdgeDeployment deployment = await this.runtime.DeployConfigurationAsync(addLoadGenConfig + addTrcConfig, token, Context.Current.NestedEdge);
  PriorityQueueTestStatus loadGenTestStatus = await this.PollUntilFinishedAsync(LoadGenModuleName, token);

diff --git a/test/Microsoft.Azure.Devices.Edge.Test/helpers/TestResultCoordinatorUtil.cs b/test/Microsoft.Azure.Devices.Edge.Test/helpers/TestResultCoordinatorUtil.cs
@@ -44,7 +44,12 @@ public static Action<EdgeConfigBuilder> BuildAddNetworkControllerConfig(string t
  });
  }
 
- public static Action<EdgeConfigBuilder> BuildAddTestResultCoordinatorConfig(string trackingId, string trcImage, string expectedSourceModuleName, string actualSourceModuleName)
+ public static Action<EdgeConfigBuilder> BuildAddTestResultCoordinatorConfig(
+ string trackingId,
+ string trcImage,
+ string expectedSourceModuleName,
+ string actualSourceModuleName,
+ bool mqttBrokerEnabled)
  {
  return new Action<EdgeConfigBuilder>(
  builder =>
@@ -61,7 +66,8 @@ public static Action<EdgeConfigBuilder> BuildAddTestResultCoordinatorConfig(stri
  ("testStartDelay", "00:00:00"),
  ("verificationDelay", "00:00:00"),
  ("NetworkControllerRunProfile", "Online"),
- ("TEST_INFO", "key=unnecessary")
+ ("TEST_INFO", "key=unnecessary"),
+ ("mqttBrokerEnabled", mqttBrokerEnabled.ToString())
  })
  .WithSettings(new[] { ("createOptions", "{\"HostConfig\": {\"PortBindings\": {\"5001/tcp\": [{\"HostPort\": \"5001\"}]}}}") })