From 33e61ead4f402cb1ce2af85e62b2d7cb9a012e4c Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Tue, 30 Nov 2021 18:32:41 +0000 Subject: [PATCH 1/8] squash dev commits --- builds/e2e/connectivity.yaml | 4 +- builds/e2e/nested-connectivity.yaml | 1 + builds/e2e/templates/connectivity-deploy.yaml | 2 + .../connectivity_deployment.template.json | 5 +- ...Deployment_connectivity_mqtt.template.json | 27 +++--- ...aseDeployment_long_haul_amqp.template.json | 3 + ...aseDeployment_long_haul_mqtt.template.json | 6 ++ scripts/linux/trcE2ETest.sh | 6 ++ .../GenericMqtt.cs | 2 +- .../PriorityQueues.cs | 6 +- .../helpers/TestResultCoordinatorUtil.cs | 10 +- .../Reports/CountingReportGeneratorTest.cs | 25 +++++ .../Reports/CountingReportTest.cs | 6 ++ ...DirectMethodLongHaulReportGeneratorTest.cs | 19 ++++ .../TwinCountingReportGeneratorTest.cs | 5 + .../TestReportUtilTest.cs | 7 +- .../Reports/CountingReport.cs | 45 +++++++-- .../Reports/CountingReportGenerator.cs | 5 + .../DirectMethodConnectivityReport.cs | 3 + .../DirectMethodConnectivityReportMetadata.cs | 4 - .../LongHaul/DirectMethodLongHaulReport.cs | 56 +++++++++-- .../DirectMethodLongHaulReportGenerator.cs | 10 ++ .../LegacyTwin/LegacyTwinReportGenerator.cs | 96 +++++++++++++++++-- .../Reports/TestReportGeneratorFactory.cs | 9 +- .../Reports/TwinCountingReport.cs | 3 + .../Reports/TwinCountingReportMetadata.cs | 3 - .../modules/TestResultCoordinator/Settings.cs | 15 ++- test/modules/TwinTester/StatusCode.cs | 2 +- 28 files changed, 325 insertions(+), 60 deletions(-) diff --git a/builds/e2e/connectivity.yaml b/builds/e2e/connectivity.yaml index 7546df55d8b..fff69c08fce 100644 --- a/builds/e2e/connectivity.yaml +++ b/builds/e2e/connectivity.yaml @@ -218,6 +218,7 @@ jobs: customEdgeHub.image: '$(customEdgeHub.image)' test.runtimeLogLevel: '$(test.runtimeLogLevel)' testInfo.testName: '$(testInfo.testName)' + topology: 'SingleNode' ################################################################################ @@ -398,4 +399,5 @@ jobs: customEdgeHub.image: '$(customEdgeHub.image)' test.runtimeLogLevel: '$(test.runtimeLogLevel)' connectivity.nested: 'false' - testInfo.testName: '$(testInfo.testName)' \ No newline at end of file + testInfo.testName: '$(testInfo.testName)' + topology: 'SingleNode' \ No newline at end of file diff --git a/builds/e2e/nested-connectivity.yaml b/builds/e2e/nested-connectivity.yaml index 2c664173c17..76c331e9a21 100644 --- a/builds/e2e/nested-connectivity.yaml +++ b/builds/e2e/nested-connectivity.yaml @@ -176,6 +176,7 @@ stages: aziotis.package.filter: '$(aziotis.package.filter)' connectivity.nested: 'true' testInfo.testName: '$(testInfo.testName)' + topology: 'Nested' - stage: Cleanup diff --git a/builds/e2e/templates/connectivity-deploy.yaml b/builds/e2e/templates/connectivity-deploy.yaml index 5b151fca049..c5394162e02 100644 --- a/builds/e2e/templates/connectivity-deploy.yaml +++ b/builds/e2e/templates/connectivity-deploy.yaml @@ -30,6 +30,7 @@ parameters: customEdgeAgent.image: '' customEdgeHub.image: '' test.runtimeLogLevel: '' + topology: '' connectivity.nested: 'false' steps: @@ -143,6 +144,7 @@ steps: -testRuntimeLogLevel "${{ parameters['test.runtimeLogLevel'] }}" \ -testInfo "$testInfo" \ -repoPath "${{ parameters['build.repo.path'] }}" \ + -topology "${{ parameters['topology'] }}" \ -waitForTestComplete \ -cleanAll diff --git a/e2e_deployment_files/connectivity_deployment.template.json b/e2e_deployment_files/connectivity_deployment.template.json index 7a8575d089e..9d18f0f4ecf 100644 --- a/e2e_deployment_files/connectivity_deployment.template.json +++ b/e2e_deployment_files/connectivity_deployment.template.json @@ -33,7 +33,7 @@ "ExperimentalFeatures__EnableUploadLogs": { "value": "true" }, - "RuntimeLogLevel" : { + "RuntimeLogLevel": { "value": "debug" } }, @@ -399,6 +399,9 @@ }, "TEST_INFO": { "value": "" + }, + "topology": { + "value": "" } }, "settings": { diff --git a/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_connectivity_mqtt.template.json b/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_connectivity_mqtt.template.json index 4d103a6d36b..b9447232c98 100644 --- a/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_connectivity_mqtt.template.json +++ b/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_connectivity_mqtt.template.json @@ -387,6 +387,12 @@ }, "TEST_INFO": { "value": "" + }, + "topology": { + "value": "Nested" + }, + "mqttBrokerEnabled": { + "value": "true" } }, "settings": { @@ -726,32 +732,28 @@ "TwinTestPropertyType": "Desired", "ExpectedSource": "twinTester1.desiredUpdated", "ActualSource": "twinTester2.desiredReceived", - "TestDescription": "twin | desired property | amqp", - "Topology": "Nested" + "TestDescription": "twin | desired property | amqp" }, "reportMetadata6": { "TestReportType": "TwinCountingReport", "TwinTestPropertyType": "Reported", "ExpectedSource": "twinTester2.reportedReceived", "ActualSource": "twinTester2.reportedUpdated", - "TestDescription": "twin | reported property | amqp", - "Topology": "Nested" + "TestDescription": "twin | reported property | amqp" }, "reportMetadata7": { "TestReportType": "TwinCountingReport", "TwinTestPropertyType": "Desired", "ExpectedSource": "twinTester3.desiredUpdated", "ActualSource": "twinTester4.desiredReceived", - "TestDescription": "twin | desired property | mqtt", - "Topology": "Nested" + "TestDescription": "twin | desired property | mqtt" }, "reportMetadata8": { "TestReportType": "TwinCountingReport", "TwinTestPropertyType": "Reported", "ExpectedSource": "twinTester4.reportedReceived", "ActualSource": "twinTester4.reportedUpdated", - "TestDescription": "twin | reported property | mqtt", - "Topology": "Nested" + "TestDescription": "twin | reported property | mqtt" }, "reportMetadata9": { "TestReportType": "DeploymentTestReport", @@ -764,23 +766,20 @@ "SenderSource": "directMethodSender1.send", "ReceiverSource": "directMethodReceiver1.receive", "TolerancePeriod": "00:01:00", - "TestDescription": "direct method | cloud | amqp", - "Topology": "Nested" + "TestDescription": "direct method | cloud | amqp" }, "reportMetadata11": { "TestReportType": "DirectMethodConnectivityReport", "SenderSource": "directMethodSender2.send", "ReceiverSource": "directMethodReceiver2.receive", "TolerancePeriod": "00:01:00", - "TestDescription": "direct method | cloud | mqtt", - "Topology": "Nested" + "TestDescription": "direct method | cloud | mqtt" }, "reportMetadata12": { "TestReportType": "DirectMethodConnectivityReport", "SenderSource": "directMethodSender3.send", "TolerancePeriod": "00:01:00", - "TestDescription": "edge agent ping", - "Topology": "Nested" + "TestDescription": "edge agent ping" }, "reportMetadata15": { "TestReportType": "NetworkControllerReport", diff --git a/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_amqp.template.json b/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_amqp.template.json index eb0480ac674..60cb33cfc30 100644 --- a/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_amqp.template.json +++ b/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_amqp.template.json @@ -399,6 +399,9 @@ }, "testMode": { "value": "" + }, + "topology": { + "value": "Nested" } }, "settings": { diff --git a/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_mqtt.template.json b/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_mqtt.template.json index a1520ad8a25..68e5580801f 100644 --- a/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_mqtt.template.json +++ b/e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_mqtt.template.json @@ -408,6 +408,12 @@ }, "testMode": { "value": "" + }, + "topology": { + "value": "Nested" + }, + "mqttBrokerEnabled": { + "value": "true" } }, "settings": { diff --git a/scripts/linux/trcE2ETest.sh b/scripts/linux/trcE2ETest.sh index 5d1daab3739..56f31fd186f 100755 --- a/scripts/linux/trcE2ETest.sh +++ b/scripts/linux/trcE2ETest.sh @@ -50,6 +50,7 @@ function usage() { echo ' -restartIntervalInMins Value for long haul specifying how often a random module will restart. If specified, then "desiredModulesToRestartCSV" must be specified as well.' echo ' -sendReportFrequency Value for long haul specifying how often TRC will send reports to LogAnalytics.' echo " -testMode Test mode for TestResultCoordinator to start up with correct settings. Value is either 'LongHaul' or 'Connectivity'." + echo " -topology Configuration telling the TRC which topology tests are running in." echo " -repoPath Path of the checked-out iotedge repository for getting the deployment file." echo " -clientModuleTransportType Value for contrained long haul specifying transport type for all client modules." echo " -trackingId Tracking id used to tag test events. Needed if running nested tests and test events are sent to TRC from L4 node. Otherwise generated." @@ -211,6 +212,7 @@ function prepare_test_from_artifacts() { sed -i -e "s@@${NETWORK_CONTROLLER_FREQUENCIES[2]}@g" "$deployment_working_file" sed -i -e "s@@$TEST_MODE@g" "$deployment_working_file" + sed -i -e "s@@$TOPOLOGY@g" "$deployment_working_file" sed -i -e "s@@$log_rotation_max_file@g" "$deployment_working_file" sed -i -e "s@@$log_rotation_max_file_edgehub@g" "$deployment_working_file" @@ -456,6 +458,9 @@ function process_args() { elif [ $saveNextArg -eq 47 ]; then TRACKING_ID="$arg" saveNextArg=0; + elif [ $saveNextArg -eq 48 ]; then + TOPOLOGY="$arg" + saveNextArg=0; else case "$arg" in '-h' | '--help' ) usage;; @@ -506,6 +511,7 @@ function process_args() { '-repoPath' ) saveNextArg=45;; '-clientModuleTransportType' ) saveNextArg=46;; '-trackingId' ) saveNextArg=47;; + '-topology' ) saveNextArg=48;; '-waitForTestComplete' ) WAIT_FOR_TEST_COMPLETE=1;; '-cleanAll' ) CLEAN_ALL=1;; diff --git a/test/Microsoft.Azure.Devices.Edge.Test/GenericMqtt.cs b/test/Microsoft.Azure.Devices.Edge.Test/GenericMqtt.cs index f6854604506..876f69e0d9f 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/GenericMqtt.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/GenericMqtt.cs @@ -47,7 +47,7 @@ public async Task GenericMqttTelemetry() Action addMqttBrokerConfig = MqttBrokerUtil.BuildAddBrokerToDeployment(false); Action addNetworkControllerConfig = TestResultCoordinatorUtil.BuildAddNetworkControllerConfig(trackingId, networkControllerImage); - Action addTestResultCoordinatorConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, GenericMqttInitiatorModuleName, GenericMqttInitiatorModuleName); + Action addTestResultCoordinatorConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, GenericMqttInitiatorModuleName, GenericMqttInitiatorModuleName, true); Action addGenericMqttTesterConfig = this.BuildAddGenericMqttTesterConfig(trackingId, trcImage, genericMqttTesterImage); Action config = addMqttBrokerConfig + addNetworkControllerConfig + addTestResultCoordinatorConfig + addGenericMqttTesterConfig; EdgeDeployment deployment = await this.runtime.DeployConfigurationAsync(config, token, Context.Current.NestedEdge); diff --git a/test/Microsoft.Azure.Devices.Edge.Test/PriorityQueues.cs b/test/Microsoft.Azure.Devices.Edge.Test/PriorityQueues.cs index 2c687aeb5b7..ab97f97a565 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/PriorityQueues.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/PriorityQueues.cs @@ -44,7 +44,7 @@ public async Task PriorityQueueModuleToModuleMessages() TestInfo testInfo = this.InitTestInfo(5, 1000, true); Action addLoadGenConfig = this.BuildAddLoadGenConfig(trackingId, loadGenImage, testInfo, false); - Action addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, RelayerModuleName); + Action addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, RelayerModuleName, false); EdgeDeployment deployment = await this.runtime.DeployConfigurationAsync(addLoadGenConfig + addTrcConfig, token, Context.Current.NestedEdge); PriorityQueueTestStatus loadGenTestStatus = await this.PollUntilFinishedAsync(LoadGenModuleName, token); @@ -69,7 +69,7 @@ public async Task PriorityQueueModuleToHubMessages() var testResultReportingClient = new TestResultReportingClient { BaseUrl = "http://localhost:5001" }; Action addLoadGenConfig = this.BuildAddLoadGenConfig(trackingId, loadGenImage, testInfo, true); - Action addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, "hubtest"); + Action addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, "hubtest", false); Action addNetworkControllerConfig = TestResultCoordinatorUtil.BuildAddNetworkControllerConfig(trackingId, networkControllerImage); EdgeDeployment deployment = await this.runtime.DeployConfigurationAsync(addLoadGenConfig + addTrcConfig + addNetworkControllerConfig, token, Context.Current.NestedEdge); @@ -100,7 +100,7 @@ public async Task PriorityQueueTimeToLive() TestInfo testInfo = this.InitTestInfo(5, 20); Action addLoadGenConfig = this.BuildAddLoadGenConfig(trackingId, loadGenImage, testInfo, false); - Action addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, RelayerModuleName); + Action addTrcConfig = TestResultCoordinatorUtil.BuildAddTestResultCoordinatorConfig(trackingId, trcImage, LoadGenModuleName, RelayerModuleName, false); EdgeDeployment deployment = await this.runtime.DeployConfigurationAsync(addLoadGenConfig + addTrcConfig, token, Context.Current.NestedEdge); PriorityQueueTestStatus loadGenTestStatus = await this.PollUntilFinishedAsync(LoadGenModuleName, token); diff --git a/test/Microsoft.Azure.Devices.Edge.Test/helpers/TestResultCoordinatorUtil.cs b/test/Microsoft.Azure.Devices.Edge.Test/helpers/TestResultCoordinatorUtil.cs index 3dfab3748e4..0abf50fdeb8 100644 --- a/test/Microsoft.Azure.Devices.Edge.Test/helpers/TestResultCoordinatorUtil.cs +++ b/test/Microsoft.Azure.Devices.Edge.Test/helpers/TestResultCoordinatorUtil.cs @@ -44,7 +44,12 @@ public static Action BuildAddNetworkControllerConfig(string t }); } - public static Action BuildAddTestResultCoordinatorConfig(string trackingId, string trcImage, string expectedSourceModuleName, string actualSourceModuleName) + public static Action BuildAddTestResultCoordinatorConfig( + string trackingId, + string trcImage, + string expectedSourceModuleName, + string actualSourceModuleName, + bool mqttBrokerEnabled) { return new Action( builder => @@ -61,7 +66,8 @@ public static Action BuildAddTestResultCoordinatorConfig(stri ("testStartDelay", "00:00:00"), ("verificationDelay", "00:00:00"), ("NetworkControllerRunProfile", "Online"), - ("TEST_INFO", "key=unnecessary") + ("TEST_INFO", "key=unnecessary"), + ("mqttBrokerEnabled", mqttBrokerEnabled.ToString()) }) .WithSettings(new[] { ("createOptions", "{\"HostConfig\": {\"PortBindings\": {\"5001/tcp\": [{\"HostPort\": \"5001\"}]}}}") }) diff --git a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs index dc900a14a39..912d384a0d2 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs @@ -49,6 +49,7 @@ public class CountingReportGeneratorTest [Fact] public void TestConstructorSuccess() { + bool brokerEnabled = false; string expectedSource = "expectedSource"; string actualSource = "actualSource"; int batchSize = 10; @@ -62,6 +63,7 @@ public void TestConstructorSuccess() var reportGenerator = new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), expectedSource, expectedResults.GetAsyncEnumerator(), @@ -86,6 +88,7 @@ public void TestConstructorSuccess() [InlineData("")] public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDescription) { + bool brokerEnabled = false; var mockExpectedResults = new Mock>(); var mockActualStore = new Mock>(); @@ -93,6 +96,7 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes () => new CountingReportGenerator( testDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", mockExpectedResults.Object, @@ -111,6 +115,7 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes [InlineData("")] public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId) { + bool brokerEnabled = false; var mockExpectedResults = new Mock>(); var mockActualStore = new Mock>(); @@ -118,6 +123,7 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId) () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, trackingId, "expectedSource", mockExpectedResults.Object, @@ -136,6 +142,7 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId) [InlineData("")] public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expectedSource) { + bool brokerEnabled = false; var mockExpectedResults = new Mock>(); var mockActualStore = new Mock>(); @@ -143,6 +150,7 @@ public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expected () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), expectedSource, mockExpectedResults.Object, @@ -159,12 +167,14 @@ public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expected [Fact] public void TestConstructorThrowsWhenExpectedStoreIsNotProvided() { + bool brokerEnabled = false; var mockActualStore = new Mock>(); ArgumentNullException ex = Assert.Throws( () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", null, @@ -183,6 +193,7 @@ public void TestConstructorThrowsWhenExpectedStoreIsNotProvided() [InlineData("")] public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSource) { + bool brokerEnabled = false; var mockExpectedResults = new Mock>(); var mockActualStore = new Mock>(); @@ -190,6 +201,7 @@ public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSour () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", mockExpectedResults.Object, @@ -206,12 +218,14 @@ public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSour [Fact] public void TestConstructorThrowsWhenActualStoreIsNotProvided() { + bool brokerEnabled = false; var mockExpectedResults = new Mock>(); ArgumentNullException ex = Assert.Throws( () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", mockExpectedResults.Object, @@ -230,6 +244,7 @@ public void TestConstructorThrowsWhenActualStoreIsNotProvided() [InlineData("")] public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType) { + bool brokerEnabled = false; var mockExpectedResults = new Mock>(); var mockActualStore = new Mock>(); @@ -237,6 +252,7 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType) () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", mockExpectedResults.Object, @@ -253,6 +269,7 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType) [Fact] public void TestConstructorThrowsWhenTestResultComparerIsNotProvided() { + bool brokerEnabled = false; var mockExpectedResults = new Mock>(); var mockActualStore = new Mock>(); @@ -260,6 +277,7 @@ public void TestConstructorThrowsWhenTestResultComparerIsNotProvided() () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", mockExpectedResults.Object, @@ -277,6 +295,7 @@ public void TestConstructorThrowsWhenTestResultComparerIsNotProvided() [InlineData(0)] public void TestConstructorThrowsWhenUnmatchedResultsMaxSizeIsNonPositive(ushort unmatchedResultsMaxSize) { + bool brokerEnabled = false; var mockExpectedResults = new Mock>(); var mockActualStore = new Mock>(); @@ -284,6 +303,7 @@ public void TestConstructorThrowsWhenUnmatchedResultsMaxSizeIsNonPositive(ushort () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", mockExpectedResults.Object, @@ -298,6 +318,7 @@ public void TestConstructorThrowsWhenUnmatchedResultsMaxSizeIsNonPositive(ushort [Fact] public async Task TestCreateReportAsyncWithEmptyResults() { + bool brokerEnabled = false; string expectedSource = "expectedSource"; string actualSource = "actualSource"; int batchSize = 10; @@ -310,6 +331,7 @@ public async Task TestCreateReportAsyncWithEmptyResults() var reportGenerator = new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), expectedSource, expectedResults.GetAsyncEnumerator(), @@ -348,6 +370,8 @@ public async Task TestCreateReportAsync( ulong expectedTotalMisorderedActualResultCount, ulong expectedMissingResultsCount) { + bool brokerEnabled = false; + // give fake tracking id and batch id to mimic real scenario expectedStoreValues = expectedStoreValues.Select(v => "xx;yy;" + v); actualStoreValues = actualStoreValues.Select(v => "xx;yy;" + v); @@ -364,6 +388,7 @@ public async Task TestCreateReportAsync( var reportGenerator = new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), expectedSource, expectedResults.GetAsyncEnumerator(), diff --git a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs index 209309bd887..d6a8632b58a 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs @@ -21,6 +21,7 @@ public void TestConstructorSuccess() var report = new CountingReport( TestDescription, TestMode.Connectivity, + false, "trackingId123", "expectedSource", "actualSource", @@ -77,6 +78,7 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes () => new CountingReport( testDescription, TestMode.Connectivity, + false, "trackingId123", "expectedSource", "actualSource", @@ -109,6 +111,7 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId) () => new CountingReport( TestDescription, TestMode.Connectivity, + false, trackingId, "expectedSource", "actualSource", @@ -141,6 +144,7 @@ public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expected () => new CountingReport( TestDescription, TestMode.Connectivity, + false, "trackingId-23434", expectedSource, "actualSource", @@ -173,6 +177,7 @@ public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSour () => new CountingReport( TestDescription, TestMode.Connectivity, + false, "trackingId-23434", "expectedSource", actualSource, @@ -205,6 +210,7 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType) () => new CountingReport( TestDescription, TestMode.Connectivity, + false, "trackingId-23434", "expectedSource", "actualSource", diff --git a/test/modules/Modules.Test/TestResultCoordinator/Reports/DirectMethod/DirectMethodLongHaulReportGeneratorTest.cs b/test/modules/Modules.Test/TestResultCoordinator/Reports/DirectMethod/DirectMethodLongHaulReportGeneratorTest.cs index 67ab8a6b5ad..192af6f6248 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/Reports/DirectMethod/DirectMethodLongHaulReportGeneratorTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/Reports/DirectMethod/DirectMethodLongHaulReportGeneratorTest.cs @@ -6,6 +6,7 @@ namespace Modules.Test.TestResultCoordinator.Reports.DirectMethod using System.Linq; using System.Net; using System.Threading.Tasks; + using global::TestResultCoordinator; using global::TestResultCoordinator.Reports; using global::TestResultCoordinator.Reports.DirectMethod.LongHaul; using Microsoft.Azure.Devices.Edge.ModuleUtil; @@ -36,6 +37,8 @@ public void TestConstructorSuccess() TestDescription, Guid.NewGuid().ToString(), senderSource, + Topology.SingleNode, + false, senderResults.GetAsyncEnumerator(), receiverSource, receiverResults.GetAsyncEnumerator(), @@ -64,6 +67,8 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes testDescription, Guid.NewGuid().ToString(), "senderSource", + Topology.SingleNode, + false, mockSenderResults.Object, "receiverSource", receiverResults.GetAsyncEnumerator(), @@ -87,6 +92,8 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId) TestDescription, trackingId, "senderSource", + Topology.SingleNode, + false, mockSenderResults.Object, "receiverSource", receiverResults.GetAsyncEnumerator(), @@ -110,6 +117,8 @@ public void TestConstructorThrowsWhenSenderSourceIsNotProvided(string senderSour TestDescription, Guid.NewGuid().ToString(), senderSource, + Topology.SingleNode, + false, mockSenderResults.Object, "receiverSource", receiverResults.GetAsyncEnumerator(), @@ -130,6 +139,8 @@ public void TestConstructorThrowsWhenSenderStoreIsNotProvided() TestDescription, Guid.NewGuid().ToString(), "senderSource", + Topology.SingleNode, + false, null, "receiverSource", receiverResults.GetAsyncEnumerator(), @@ -153,6 +164,8 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType) TestDescription, Guid.NewGuid().ToString(), "senderSource", + Topology.SingleNode, + false, mockSenderResults.Object, "receiverSource", receiverResults.GetAsyncEnumerator(), @@ -191,6 +204,8 @@ public async Task TestCreateReportAsync( TestDescription, Guid.NewGuid().ToString(), senderSource, + Topology.SingleNode, + false, senderResults.GetAsyncEnumerator(), receiverSource, receiverResults.GetAsyncEnumerator(), @@ -254,6 +269,8 @@ public async Task TestOtherStatusCodeCounts() TestDescription, Guid.NewGuid().ToString(), senderSource, + Topology.SingleNode, + false, senderResults.GetAsyncEnumerator(), receiverSource, receiverResults.GetAsyncEnumerator(), @@ -303,6 +320,8 @@ public async Task TestCreateReportAsyncWithEmptyResults() TestDescription, Guid.NewGuid().ToString(), senderSource, + Topology.SingleNode, + false, senderResults.GetAsyncEnumerator(), receiverSource, receiverResults.GetAsyncEnumerator(), diff --git a/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs b/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs index d34e86acef7..14d5d23b1c3 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs @@ -185,6 +185,8 @@ public void TestConstructorThrowsWhenActualStoreIsNotProvided() [InlineData("")] public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType) { + bool brokerEnabled = false; + var mockExpectedResults = new Mock>(); var mockActualStore = new Mock>(); @@ -192,6 +194,7 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType) () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", mockExpectedResults.Object, @@ -251,6 +254,7 @@ public void TestConstructorThrowsWhenUnmatchedResultsMaxSizeIsNonPositive(ushort [Fact] public async Task TestCreateReportAsyncWithEmptyResults() { + bool brokerEnabled = false; string expectedSource = "expectedSource"; string actualSource = "actualSource"; int batchSize = 10; @@ -263,6 +267,7 @@ public async Task TestCreateReportAsyncWithEmptyResults() var reportGenerator = new CountingReportGenerator( TestDescription, TestMode.Connectivity, + brokerEnabled, Guid.NewGuid().ToString(), expectedSource, expectedResults.GetAsyncEnumerator(), diff --git a/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs b/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs index 8c5361ff02f..d6713fd6de5 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs @@ -79,10 +79,10 @@ public async Task TestGenerateTestResultReportsAsync_ReportGeneration( string trackingId = "fakeTrackingId"; var countingReportMetadata = new CountingReportMetadata(TestDescription, "CountingExpectedSource", "CountingAcutalSource", TestOperationResultType.Messages, TestReportType.CountingReport, false); - var twinCountingReportMetadata = new TwinCountingReportMetadata(TestDescription, Topology.SingleNode, "TwinExpectedSource", "TwinActualSource", TestReportType.TwinCountingReport, TwinTestPropertyType.Desired); + var twinCountingReportMetadata = new TwinCountingReportMetadata(TestDescription, "TwinExpectedSource", "TwinActualSource", TestReportType.TwinCountingReport, TwinTestPropertyType.Desired); var deploymentReportMetadata = new DeploymentTestReportMetadata(TestDescription, "DeploymentExpectedSource", "DeploymentActualSource"); - var directMethodConnectivityReportMetadata = new DirectMethodConnectivityReportMetadata(TestDescription, Topology.SingleNode, "DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource"); - var directMethodConnectivityReportMetadataWithoutReceiverSource = new DirectMethodConnectivityReportMetadata(TestDescription, Topology.SingleNode, "DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource"); + var directMethodConnectivityReportMetadata = new DirectMethodConnectivityReportMetadata(TestDescription, "DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource"); + var directMethodConnectivityReportMetadataWithoutReceiverSource = new DirectMethodConnectivityReportMetadata(TestDescription, "DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource"); var directMethodLongHaulReportMetadata = new DirectMethodLongHaulReportMetadata(TestDescription, "DirectMethodSenderSource", "DirectMethodReceiverSource"); var directMethodLongHaulReportMetadataWithoutReceiverSource = new DirectMethodLongHaulReportMetadata(TestDescription, "DirectMethodSenderSource", "DirectMethodReceiverSource"); var edgeHubRestartMessageReportMetadata = new EdgeHubRestartMessageReportMetadata(TestDescription, "edgeHubRestartTester1.EdgeHubRestartMessage", "relayer1.receive"); @@ -543,6 +543,7 @@ private Task MockTestResultReport(bool throwException) return Task.FromResult(new CountingReport( "mock", TestMode.Connectivity, + false, "mock", "mock", "mock", diff --git a/test/modules/TestResultCoordinator/Reports/CountingReport.cs b/test/modules/TestResultCoordinator/Reports/CountingReport.cs index 0d1fea056a7..b8a47f688b3 100644 --- a/test/modules/TestResultCoordinator/Reports/CountingReport.cs +++ b/test/modules/TestResultCoordinator/Reports/CountingReport.cs @@ -7,6 +7,7 @@ namespace TestResultCoordinator.Reports using Microsoft.Azure.Devices.Edge.Util; using Microsoft.Azure.Devices.Edge.Util.Json; using Newtonsoft.Json; + using Newtonsoft.Json.Converters; /// /// This is a counting report to show test result counts. It tracks a number @@ -28,11 +29,13 @@ namespace TestResultCoordinator.Reports class CountingReport : TestResultReportBase { const string C2dTestDescription = "C2D"; - const string GenericMqttTelemetryTestDescription = "messages | local | mqtt | generic"; + const string GenericMqttTelemetryTestDescription = "mqtt | generic"; + const string UpstreamMessagesTestDescription = "messages | upstream"; public CountingReport( string testDescription, TestMode testMode, + bool mqttBrokerEnabled, string trackingId, string expectedSource, string actualSource, @@ -51,6 +54,7 @@ public CountingReport( : base(testDescription, trackingId, resultType) { this.TestMode = testMode; + this.MqttBrokerEnabled = mqttBrokerEnabled; this.ExpectedSource = Preconditions.CheckNonWhiteSpace(expectedSource, nameof(expectedSource)); this.ActualSource = Preconditions.CheckNonWhiteSpace(actualSource, nameof(actualSource)); this.TotalExpectCount = totalExpectCount; @@ -67,8 +71,11 @@ public CountingReport( this.LastActualResultTimestamp = lastActualResultTimestamp; } + [JsonConverter(typeof(StringEnumConverter))] public TestMode TestMode { get; } + public bool MqttBrokerEnabled { get; } + public string ExpectedSource { get; } public string ActualSource { get; } @@ -115,19 +122,37 @@ bool IsPassedHelper() }, () => { - // Product issue for C2D messages connected to edgehub over mqtt. - // We should remove this failure tolerance when fixed. - if (this.TestDescription.Contains(C2dTestDescription)) - { - return ((double)this.TotalMatchCount / this.TotalExpectCount) > .8d; - } - else if (this.TestDescription == GenericMqttTelemetryTestDescription && this.TestMode == TestMode.Connectivity) + if (this.TestMode == TestMode.Connectivity) { - return ((double)this.TotalMatchCount / this.TotalExpectCount) > .9d; + // Product issue for C2D messages connected to edgehub. + // We should remove this failure tolerance when fixed. + if (this.TestDescription.Contains(C2dTestDescription)) + { + return ((double)this.TotalMatchCount / this.TotalExpectCount) > .8d; + } + else if (this.TestDescription == GenericMqttTelemetryTestDescription) + { + return ((double)this.TotalMatchCount / this.TotalExpectCount) > .9d; + } + else + { + return this.TotalExpectCount == this.TotalMatchCount; + } } else { - return this.TotalExpectCount == this.TotalMatchCount; + if (this.TestDescription.Contains(GenericMqttTelemetryTestDescription)) + { + return ((double)this.TotalMatchCount / this.TotalExpectCount) > .8d; + } + else if (this.MqttBrokerEnabled && this.TestDescription.Contains(UpstreamMessagesTestDescription)) + { + return ((double)this.TotalMatchCount / this.TotalExpectCount) > .99d; + } + else + { + return this.TotalExpectCount == this.TotalMatchCount; + } } }); } diff --git a/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs b/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs index 3de0be6b002..ec99f1e9541 100644 --- a/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs +++ b/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs @@ -23,6 +23,7 @@ sealed class CountingReportGenerator : ITestResultReportGenerator internal CountingReportGenerator( string testDescription, TestMode testMode, + bool mqttBrokerEnabled, string trackingId, string expectedSource, IAsyncEnumerator expectedTestResults, @@ -35,6 +36,7 @@ internal CountingReportGenerator( { this.TestDescription = Preconditions.CheckNonWhiteSpace(testDescription, nameof(testDescription)); this.TestMode = testMode; + this.MqttBrokerEnabled = mqttBrokerEnabled; this.trackingId = Preconditions.CheckNonWhiteSpace(trackingId, nameof(trackingId)); this.ExpectedTestResults = Preconditions.CheckNotNull(expectedTestResults, nameof(expectedTestResults)); this.ExpectedSource = Preconditions.CheckNonWhiteSpace(expectedSource, nameof(expectedSource)); @@ -48,6 +50,8 @@ internal CountingReportGenerator( internal TestMode TestMode { get; } + internal bool MqttBrokerEnabled { get; } + internal string ActualSource { get; } internal IAsyncEnumerator ActualTestResults { get; } @@ -236,6 +240,7 @@ public async Task CreateReportAsync() return new CountingReport( this.TestDescription, this.TestMode, + this.MqttBrokerEnabled, this.trackingId, this.ExpectedSource, this.ActualSource, diff --git a/test/modules/TestResultCoordinator/Reports/DirectMethod/Connectivity/DirectMethodConnectivityReport.cs b/test/modules/TestResultCoordinator/Reports/DirectMethod/Connectivity/DirectMethodConnectivityReport.cs index e0f6d05ba79..f3b639646c8 100644 --- a/test/modules/TestResultCoordinator/Reports/DirectMethod/Connectivity/DirectMethodConnectivityReport.cs +++ b/test/modules/TestResultCoordinator/Reports/DirectMethod/Connectivity/DirectMethodConnectivityReport.cs @@ -1,7 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. namespace TestResultCoordinator.Reports.DirectMethod.Connectivity { + using System.Text.Json.Serialization; using Microsoft.Azure.Devices.Edge.Util; + using Newtonsoft.Json.Converters; using TestResultCoordinator.Reports; /// @@ -39,6 +41,7 @@ public DirectMethodConnectivityReport( this.MismatchFailure = mismatchFailure; } + [JsonConverter(typeof(StringEnumConverter))] public Topology Topology { get; } public string SenderSource { get; } diff --git a/test/modules/TestResultCoordinator/Reports/DirectMethod/Connectivity/DirectMethodConnectivityReportMetadata.cs b/test/modules/TestResultCoordinator/Reports/DirectMethod/Connectivity/DirectMethodConnectivityReportMetadata.cs index acb90768442..b984018273d 100644 --- a/test/modules/TestResultCoordinator/Reports/DirectMethod/Connectivity/DirectMethodConnectivityReportMetadata.cs +++ b/test/modules/TestResultCoordinator/Reports/DirectMethod/Connectivity/DirectMethodConnectivityReportMetadata.cs @@ -10,7 +10,6 @@ class DirectMethodConnectivityReportMetadata : TestReportMetadataBase, ITestRepo { public DirectMethodConnectivityReportMetadata( string testDescription, - Topology topology, string senderSource, TimeSpan tolerancePeriod, string receiverSource = "") @@ -19,7 +18,6 @@ public DirectMethodConnectivityReportMetadata( this.SenderSource = senderSource; this.TolerancePeriod = tolerancePeriod; this.ReceiverSource = string.IsNullOrEmpty(receiverSource) ? Option.None() : Option.Some(receiverSource); - this.Topology = topology; } public TimeSpan TolerancePeriod { get; } @@ -28,8 +26,6 @@ public DirectMethodConnectivityReportMetadata( public Option ReceiverSource { get; } - public Topology Topology; - public string[] ResultSources => this.ReceiverSource.HasValue ? new string[] { this.SenderSource, this.ReceiverSource.OrDefault() } : new string[] { this.SenderSource }; diff --git a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs index d7b07f5840c..0d8ba1a9190 100644 --- a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs +++ b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs @@ -4,7 +4,9 @@ namespace TestResultCoordinator.Reports.DirectMethod.LongHaul using System.Collections.Generic; using System.Linq; using System.Net; + using System.Text.Json.Serialization; using Microsoft.Azure.Devices.Edge.Util; + using Newtonsoft.Json.Converters; class DirectMethodLongHaulReport : TestResultReportBase { @@ -14,6 +16,8 @@ public DirectMethodLongHaulReport( string senderSource, string receiverSource, string resultType, + Topology topology, + bool mqttBrokerEnabled, long senderSuccesses, long receiverSuccesses, long statusCodeZero, @@ -25,6 +29,8 @@ public DirectMethodLongHaulReport( : base(testDescription, trackingId, resultType) { this.SenderSource = Preconditions.CheckNonWhiteSpace(senderSource, nameof(senderSource)); + this.Topology = topology; + this.MqttBrokerEnabled = mqttBrokerEnabled; this.ReceiverSource = receiverSource; this.SenderSuccesses = senderSuccesses; this.ReceiverSuccesses = receiverSuccesses; @@ -37,14 +43,28 @@ public DirectMethodLongHaulReport( } public string SenderSource { get; } + + [JsonConverter(typeof(StringEnumConverter))] + public Topology Topology { get; } + + public bool MqttBrokerEnabled { get; } + public string ReceiverSource { get; } + public long SenderSuccesses { get; } + public long ReceiverSuccesses { get; } + public long StatusCodeZero { get; } + public long Unauthorized { get; } + public long DeviceNotFound { get; } + public long TransientError { get; } + public long ResourceError { get; } + public Dictionary Other { get; } public override string Title => $"DirectMethod LongHaul Report for [{this.SenderSource}] and [{this.ReceiverSource}] ({this.ResultType})"; @@ -60,17 +80,41 @@ bool IsPassedHelper() } bool senderAndReceiverSuccessesPass = this.SenderSuccesses <= this.ReceiverSuccesses; + long allStatusCount = this.SenderSuccesses + this.StatusCodeZero + this.Other.Sum(x => x.Value); + + double statusCodeZeroThreshold; + double unauthorizedThreshold; + double deviceNotFoundThreshold; + double transientErrorThreshold; + double resourceErrorThreshold; // The SDK does not allow edgehub to de-register from iothub subscriptions, which results in DirectMethod clients sometimes receiving status code 0. // Github issue: https://github.com/Azure/iotedge/issues/681 // We expect to get this status sometimes because of edgehub restarts, but if we receive too many we should fail the tests. // TODO: When the SDK allows edgehub to de-register from subscriptions and we make the fix in edgehub, then we can fail tests for any status code 0. - long allStatusCount = this.SenderSuccesses + this.StatusCodeZero + this.Other.Sum(x => x.Value); - bool statusCodeZeroBelowThreshold = (this.StatusCodeZero == 0) || (this.StatusCodeZero < ((double)allStatusCount / 1000)); - bool unauthorizedBelowThreshold = (this.Unauthorized == 0) || (this.Unauthorized < ((double)allStatusCount / 1000)); - bool deviceNotFoundBelowThreshold = (this.DeviceNotFound == 0) || (this.DeviceNotFound < ((double)allStatusCount / 100)); - bool transientErrorBelowThreshold = (this.TransientError == 0) || (this.TransientError < ((double)allStatusCount / 100)); - bool resourceErrorBelowThreshold = (this.ResourceError == 0) || (this.ResourceError < ((double)allStatusCount / 100)); + statusCodeZeroThreshold = (double)allStatusCount / 1000; + unauthorizedThreshold = (double)allStatusCount / 1000; + transientErrorThreshold = (double)allStatusCount / 1000; + resourceErrorThreshold = (double)allStatusCount / 1000; + + if (!this.MqttBrokerEnabled && this.Topology == Topology.SingleNode) + { + deviceNotFoundThreshold = (double)allStatusCount / 1000; + } + else if (!this.MqttBrokerEnabled && this.Topology == Topology.Nested && this.TestDescription.Contains("mqtt")) + { + deviceNotFoundThreshold = (double)allStatusCount / 250; + } + else + { + deviceNotFoundThreshold = (double)allStatusCount / 100; + } + + bool statusCodeZeroBelowThreshold = (this.StatusCodeZero == 0) || (this.StatusCodeZero < statusCodeZeroThreshold); + bool unauthorizedBelowThreshold = (this.Unauthorized == 0) || (this.Unauthorized < unauthorizedThreshold); + bool deviceNotFoundBelowThreshold = (this.DeviceNotFound == 0) || (this.DeviceNotFound < deviceNotFoundThreshold); + bool transientErrorBelowThreshold = (this.TransientError == 0) || (this.TransientError < transientErrorThreshold); + bool resourceErrorBelowThreshold = (this.ResourceError == 0) || (this.ResourceError < resourceErrorThreshold); // Pass if below the thresholds, and sender and receiver got same amount of successess (or receiver has no results) return statusCodeZeroBelowThreshold && unauthorizedBelowThreshold && deviceNotFoundBelowThreshold && transientErrorBelowThreshold && senderAndReceiverSuccessesPass; diff --git a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReportGenerator.cs b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReportGenerator.cs index 5a9cf704ce0..616244432cc 100644 --- a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReportGenerator.cs +++ b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReportGenerator.cs @@ -22,6 +22,8 @@ internal DirectMethodLongHaulReportGenerator( string testDescription, string trackingId, string senderSource, + Topology topology, + bool mqttBrokerEnabled, IAsyncEnumerator senderTestResults, string receiverSource, IAsyncEnumerator receiverTestResults, @@ -30,6 +32,8 @@ internal DirectMethodLongHaulReportGenerator( this.TestDescription = Preconditions.CheckNonWhiteSpace(testDescription, nameof(testDescription)); this.trackingId = Preconditions.CheckNonWhiteSpace(trackingId, nameof(trackingId)); this.SenderSource = Preconditions.CheckNonWhiteSpace(senderSource, nameof(senderSource)); + this.Topology = topology; + this.MqttBrokerEnabled = mqttBrokerEnabled; this.SenderTestResults = Preconditions.CheckNotNull(senderTestResults, nameof(senderTestResults)); this.ReceiverSource = receiverSource; this.ReceiverTestResults = receiverTestResults; @@ -50,6 +54,10 @@ internal DirectMethodLongHaulReportGenerator( internal ITestResultComparer TestResultComparer { get; } + internal Topology Topology { get; } + + internal bool MqttBrokerEnabled { get; } + public async Task CreateReportAsync() { long statusCodeZero = 0; @@ -115,6 +123,8 @@ public async Task CreateReportAsync() this.SenderSource, this.ReceiverSource, this.ResultType, + this.Topology, + this.MqttBrokerEnabled, senderSuccesses, receiverSuccesses, statusCodeZero, diff --git a/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs b/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs index f32e5da26fc..9c43bf76cbe 100644 --- a/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs +++ b/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs @@ -2,6 +2,7 @@ namespace TestResultCoordinator.Reports.LegacyTwin { using System.Collections.Generic; + using System.Linq; using System.Threading.Tasks; using Microsoft.Azure.Devices.Edge.ModuleUtil; using Microsoft.Azure.Devices.Edge.Util; @@ -10,8 +11,9 @@ namespace TestResultCoordinator.Reports.LegacyTwin sealed class LegacyTwinReportGenerator : ITestResultReportGenerator { + const double BigToleranceProportion = .005; + const double LittleToleranceProportion = .001; static readonly ILogger Logger = ModuleUtil.CreateLogger(nameof(LegacyTwinReportGenerator)); - readonly string trackingId; internal LegacyTwinReportGenerator( @@ -19,6 +21,8 @@ internal LegacyTwinReportGenerator( string trackingId, string resultType, string senderSource, + Topology topology, + bool mqttBrokerEnabled, IAsyncEnumerator senderTestResults) { this.TestDescription = Preconditions.CheckNonWhiteSpace(testDescription, nameof(testDescription)); @@ -26,6 +30,7 @@ internal LegacyTwinReportGenerator( this.ResultType = Preconditions.CheckNonWhiteSpace(resultType, nameof(resultType)); this.SenderSource = Preconditions.CheckNonWhiteSpace(senderSource, nameof(senderSource)); this.SenderTestResults = Preconditions.CheckNotNull(senderTestResults, nameof(senderTestResults)); + this.Topology = topology; } internal string TestDescription { get; } @@ -36,19 +41,17 @@ internal LegacyTwinReportGenerator( internal IAsyncEnumerator SenderTestResults { get; } + internal Topology Topology { get; } + + internal bool MqttBrokerEnabled { get; } + public async Task CreateReportAsync() { Logger.LogInformation($"Start to generate report by {nameof(LegacyTwinReportGenerator)} for Sources [{this.SenderSource}] "); IDictionary results = new Dictionary(); - bool isPassed = true; while (await this.SenderTestResults.MoveNextAsync()) { int status = int.Parse(this.SenderTestResults.Current.Result.Substring(0, 3)); - if (status > 299) - { - isPassed = false; - } - if (results.ContainsKey(status)) { results[status] = results[status] + 1; @@ -59,6 +62,8 @@ public async Task CreateReportAsync() } } + bool isPassed = this.IsPassed(results); + var report = new LegacyTwinReport( this.TestDescription, this.trackingId, @@ -70,5 +75,82 @@ public async Task CreateReportAsync() Logger.LogInformation($"Successfully finished creating LegacyTwinReport for Source [{this.SenderSource}]"); return report; } + + // See TwinTester/StatusCode.cs for reference. + bool IsPassed(IDictionary results) + { + bool isPassed = true; + int totalResults = results.Sum(x => x.Value); + + if (totalResults == 0) + { + return false; + } + + if (this.Topology == Topology.Nested) + { + if (this.MqttBrokerEnabled || this.TestDescription.Contains("amqp")) + { + int[] bigToleranceStatusCodes = { }; + int[] littleToleranceStatusCodes = { 501, 504 }; + isPassed = this.GeneratePassResult(results, bigToleranceStatusCodes, littleToleranceStatusCodes); + } + else + { + int[] bigToleranceStatusCodes = { 505, 503, 506 }; + int[] littleToleranceStatusCodes = { 501, 502, 504 }; + isPassed = this.GeneratePassResult(results, bigToleranceStatusCodes, littleToleranceStatusCodes); + } + } + else + { + List statusCodes = (List)results.Keys; + IEnumerable failingStatusCodes = statusCodes.Where(s => + { + string statusCode = s.ToString(); + return !statusCode.StartsWith("2"); + }); + + isPassed = failingStatusCodes.Count() == 0; + } + + return isPassed; + } + + bool GeneratePassResult(IDictionary results, int[] bigToleranceStatusCodes, int[] littleToleranceStatusCodes) + { + int totalResults = results.Sum(x => x.Value); + foreach (KeyValuePair statusCodeToCount in results) + { + int statusCode = statusCodeToCount.Key; + int statusCodeCount = statusCodeToCount.Value; + + // ignore the status codes indicating some success + if (statusCode.ToString().StartsWith("2")) + { + continue; + } + else if (bigToleranceStatusCodes.Contains(statusCode)) + { + if ((double)statusCodeCount / totalResults > BigToleranceProportion) + { + return false; + } + } + else if (littleToleranceStatusCodes.Contains(statusCode)) + { + if ((double)statusCodeCount / totalResults > LittleToleranceProportion) + { + return false; + } + } + else + { + return false; + } + } + + return true; + } } } diff --git a/test/modules/TestResultCoordinator/Reports/TestReportGeneratorFactory.cs b/test/modules/TestResultCoordinator/Reports/TestReportGeneratorFactory.cs index fe50cf84bbd..c0ca8bc9af7 100644 --- a/test/modules/TestResultCoordinator/Reports/TestReportGeneratorFactory.cs +++ b/test/modules/TestResultCoordinator/Reports/TestReportGeneratorFactory.cs @@ -56,6 +56,7 @@ await this.LonghaulSettings.ForEachAsync(async (longhaulSettings) => return new CountingReportGenerator( metadata.TestDescription, Settings.Current.TestMode, + Settings.Current.MqttBrokerEnabled, trackingId, metadata.ExpectedSource, expectedTestResults.GetAsyncEnumerator(), @@ -75,7 +76,7 @@ await this.LonghaulSettings.ForEachAsync(async (longhaulSettings) => return new TwinCountingReportGenerator( metadata.TestDescription, - metadata.Topology, + Settings.Current.Topology, trackingId, metadata.ExpectedSource, expectedTestResults.GetAsyncEnumerator(), @@ -96,6 +97,8 @@ await this.LonghaulSettings.ForEachAsync(async (longhaulSettings) => trackingId, testReportMetadata.TestOperationResultType.ToString(), metadata.SenderSource, + Settings.Current.Topology, + Settings.Current.MqttBrokerEnabled, testResults.GetAsyncEnumerator()); } @@ -125,7 +128,7 @@ await this.LonghaulSettings.ForEachAsync(async (longhaulSettings) => return new DirectMethodConnectivityReportGenerator( metadata.TestDescription, - metadata.Topology, + Settings.Current.Topology, trackingId, metadata.SenderSource, senderTestResults.GetAsyncEnumerator(), @@ -153,6 +156,8 @@ await this.LonghaulSettings.ForEachAsync(async (longhaulSettings) => metadata.TestDescription, trackingId, metadata.SenderSource, + Settings.Current.Topology, + Settings.Current.MqttBrokerEnabled, senderTestResults.GetAsyncEnumerator(), metadata.ReceiverSource, receiverTestResults.GetAsyncEnumerator(), diff --git a/test/modules/TestResultCoordinator/Reports/TwinCountingReport.cs b/test/modules/TestResultCoordinator/Reports/TwinCountingReport.cs index ed65fae4771..b7b26ccdf5a 100644 --- a/test/modules/TestResultCoordinator/Reports/TwinCountingReport.cs +++ b/test/modules/TestResultCoordinator/Reports/TwinCountingReport.cs @@ -2,7 +2,9 @@ namespace TestResultCoordinator.Reports { using System.Collections.ObjectModel; + using System.Text.Json.Serialization; using Microsoft.Azure.Devices.Edge.Util; + using Newtonsoft.Json.Converters; class TwinCountingReport : TestResultReportBase { @@ -19,6 +21,7 @@ public TwinCountingReport(string testDescription, Topology topology, string trac this.UnmatchedResults = unmatchedResults; } + [JsonConverter(typeof(StringEnumConverter))] public Topology Topology { get; } public string ExpectedSource { get; } diff --git a/test/modules/TestResultCoordinator/Reports/TwinCountingReportMetadata.cs b/test/modules/TestResultCoordinator/Reports/TwinCountingReportMetadata.cs index eed653ba484..770e5cfbcee 100644 --- a/test/modules/TestResultCoordinator/Reports/TwinCountingReportMetadata.cs +++ b/test/modules/TestResultCoordinator/Reports/TwinCountingReportMetadata.cs @@ -7,7 +7,6 @@ class TwinCountingReportMetadata : CountingReportMetadata { public TwinCountingReportMetadata( string testDescription, - Topology topology, string expectedSource, string actualSource, TestReportType testReportType, @@ -15,10 +14,8 @@ public TwinCountingReportMetadata( : base(testDescription, expectedSource, actualSource, TestOperationResultType.Twin, testReportType, false) { this.TwinTestPropertyType = twinTestPropertyType; - this.Topology = topology; } public TwinTestPropertyType TwinTestPropertyType { get; } - public Topology Topology { get; } } } diff --git a/test/modules/TestResultCoordinator/Settings.cs b/test/modules/TestResultCoordinator/Settings.cs index aef696d9735..983898c8eed 100644 --- a/test/modules/TestResultCoordinator/Settings.cs +++ b/test/modules/TestResultCoordinator/Settings.cs @@ -2,7 +2,6 @@ namespace TestResultCoordinator { using System; - using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; @@ -52,6 +51,8 @@ class Settings ushort unmatchedResultsMaxSize, string testInfo, TestMode testMode, + Topology topology, + bool mqttBrokerEnabled, TimeSpan unmatchedResultTolerance, TimeSpan eventHubDelayTolerance) { @@ -118,6 +119,8 @@ class Settings this.TestInfo = ModuleUtil.ParseKeyValuePairs(testInfo, Logger, true); this.TestInfo.Add("DeviceId", this.DeviceId); this.TestMode = testMode; + this.Topology = topology; + this.MqttBrokerEnabled = mqttBrokerEnabled; } private NetworkControllerType GetNetworkControllerType(string networkControllerRunProfileName) @@ -166,6 +169,8 @@ static Settings Create() configuration.GetValue("UNMATCHED_RESULTS_MAX_SIZE", DefaultUnmatchedResultsMaxSize), configuration.GetValue("TEST_INFO"), configuration.GetValue("testMode", TestMode.Connectivity), + configuration.GetValue("topology", Topology.SingleNode), + configuration.GetValue("mqttBrokerEnabled", false), configuration.GetValue("unmatchedResultTolerance", TimeSpan.FromMinutes(1)), configuration.GetValue("eventHubDelayTolerance", TimeSpan.FromHours(1))); } @@ -204,6 +209,10 @@ static Settings Create() public TestMode TestMode { get; } + public Topology Topology { get; } + + public bool MqttBrokerEnabled { get; } + public override string ToString() { // serializing in this pattern so that secrets don't accidentally get added anywhere in the future @@ -218,7 +227,9 @@ public override string ToString() { nameof(this.TestStartDelay), this.TestStartDelay.ToString() }, { nameof(this.NetworkControllerType), this.NetworkControllerType.ToString() }, { nameof(this.TestInfo), JsonConvert.SerializeObject(this.TestInfo) }, - { nameof(this.TestMode), this.TestMode.ToString() } + { nameof(this.TestMode), this.TestMode.ToString() }, + { nameof(this.Topology), this.Topology.ToString() }, + { nameof(this.MqttBrokerEnabled), this.MqttBrokerEnabled.ToString() } }; this.TestResultEventReceivingServiceSettings.ForEach(settings => fields.Add(nameof(settings.ConsumerGroupName), settings.ConsumerGroupName)); diff --git a/test/modules/TwinTester/StatusCode.cs b/test/modules/TwinTester/StatusCode.cs index eb08f9f2576..68b6d188bd6 100644 --- a/test/modules/TwinTester/StatusCode.cs +++ b/test/modules/TwinTester/StatusCode.cs @@ -14,6 +14,6 @@ enum StatusCode ReportedPropertyUpdateCallFailure = 504, ReportedPropertyUpdateNotInCloudTwin = 505, ReportedPropertyUpdateNotInCloudTwinAfterCleanUpThreshold = 506, - ReportedPropertyReceivedAfterThreshold = 507, + ReportedPropertyReceivedAfterThreshold = 205, } } From eea62fd1f8b2a4f6d739b8d99e71be4f7095efc9 Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Thu, 2 Dec 2021 21:37:16 +0000 Subject: [PATCH 2/8] fix error in twin report generation --- .../Reports/LegacyTwin/LegacyTwinReportGenerator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs b/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs index 9c43bf76cbe..8eb7c101816 100644 --- a/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs +++ b/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs @@ -104,7 +104,7 @@ bool IsPassed(IDictionary results) } else { - List statusCodes = (List)results.Keys; + List statusCodes = results.Keys.ToList(); IEnumerable failingStatusCodes = statusCodes.Where(s => { string statusCode = s.ToString(); From f6083f1a712b7cd9199b16f31eaef51e736431cc Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Thu, 2 Dec 2021 21:47:24 +0000 Subject: [PATCH 3/8] adjust single node dm tolerance --- .../Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs index 0d8ba1a9190..1d5ecee2835 100644 --- a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs +++ b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs @@ -99,7 +99,7 @@ bool IsPassedHelper() if (!this.MqttBrokerEnabled && this.Topology == Topology.SingleNode) { - deviceNotFoundThreshold = (double)allStatusCount / 1000; + deviceNotFoundThreshold = (double)allStatusCount / 1600; } else if (!this.MqttBrokerEnabled && this.Topology == Topology.Nested && this.TestDescription.Contains("mqtt")) { From e8fbe23f6c95a5492ace8a706189b8dbb7f7c42f Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Fri, 3 Dec 2021 18:18:01 +0000 Subject: [PATCH 4/8] adjust direct method tolerance for single node --- .../Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs index 1d5ecee2835..adbb8911fbb 100644 --- a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs +++ b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs @@ -99,7 +99,7 @@ bool IsPassedHelper() if (!this.MqttBrokerEnabled && this.Topology == Topology.SingleNode) { - deviceNotFoundThreshold = (double)allStatusCount / 1600; + deviceNotFoundThreshold = (double)allStatusCount / 400; } else if (!this.MqttBrokerEnabled && this.Topology == Topology.Nested && this.TestDescription.Contains("mqtt")) { From 68a3eeb52de463bac63ad27fdb423ef43d4ecd45 Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Fri, 3 Dec 2021 18:31:53 +0000 Subject: [PATCH 5/8] broaden tolerance for messages in nested broker case --- test/modules/TestResultCoordinator/Reports/CountingReport.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/modules/TestResultCoordinator/Reports/CountingReport.cs b/test/modules/TestResultCoordinator/Reports/CountingReport.cs index b8a47f688b3..7b033196e29 100644 --- a/test/modules/TestResultCoordinator/Reports/CountingReport.cs +++ b/test/modules/TestResultCoordinator/Reports/CountingReport.cs @@ -30,7 +30,7 @@ class CountingReport : TestResultReportBase { const string C2dTestDescription = "C2D"; const string GenericMqttTelemetryTestDescription = "mqtt | generic"; - const string UpstreamMessagesTestDescription = "messages | upstream"; + const string MessagesTestDescription = "messages"; public CountingReport( string testDescription, @@ -145,7 +145,7 @@ bool IsPassedHelper() { return ((double)this.TotalMatchCount / this.TotalExpectCount) > .8d; } - else if (this.MqttBrokerEnabled && this.TestDescription.Contains(UpstreamMessagesTestDescription)) + else if (this.MqttBrokerEnabled && this.TestDescription.Contains(MessagesTestDescription)) { return ((double)this.TotalMatchCount / this.TotalExpectCount) > .99d; } From d8c6c50511c9053a69197082c1b81345de37ad3e Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Sat, 4 Dec 2021 00:02:03 +0000 Subject: [PATCH 6/8] tolerance for notimpl --- .../DirectMethod/LongHaul/DirectMethodLongHaulReport.cs | 9 ++++++++- .../LongHaul/DirectMethodLongHaulReportGenerator.cs | 5 +++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs index adbb8911fbb..4ea23b03bee 100644 --- a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs +++ b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs @@ -25,6 +25,7 @@ public DirectMethodLongHaulReport( long deviceNotFound, long transientError, long resourceError, + long notImplemented, Dictionary other) : base(testDescription, trackingId, resultType) { @@ -39,6 +40,7 @@ public DirectMethodLongHaulReport( this.DeviceNotFound = deviceNotFound; this.TransientError = transientError; this.ResourceError = resourceError; + this.NotImplemented = notImplemented; this.Other = other; } @@ -65,6 +67,8 @@ public DirectMethodLongHaulReport( public long ResourceError { get; } + public long NotImplemented { get; } + public Dictionary Other { get; } public override string Title => $"DirectMethod LongHaul Report for [{this.SenderSource}] and [{this.ReceiverSource}] ({this.ResultType})"; @@ -87,6 +91,7 @@ bool IsPassedHelper() double deviceNotFoundThreshold; double transientErrorThreshold; double resourceErrorThreshold; + double notImplementedThreshold; // The SDK does not allow edgehub to de-register from iothub subscriptions, which results in DirectMethod clients sometimes receiving status code 0. // Github issue: https://github.com/Azure/iotedge/issues/681 @@ -96,6 +101,7 @@ bool IsPassedHelper() unauthorizedThreshold = (double)allStatusCount / 1000; transientErrorThreshold = (double)allStatusCount / 1000; resourceErrorThreshold = (double)allStatusCount / 1000; + notImplementedThreshold = (double)allStatusCount / 1000; if (!this.MqttBrokerEnabled && this.Topology == Topology.SingleNode) { @@ -115,9 +121,10 @@ bool IsPassedHelper() bool deviceNotFoundBelowThreshold = (this.DeviceNotFound == 0) || (this.DeviceNotFound < deviceNotFoundThreshold); bool transientErrorBelowThreshold = (this.TransientError == 0) || (this.TransientError < transientErrorThreshold); bool resourceErrorBelowThreshold = (this.ResourceError == 0) || (this.ResourceError < resourceErrorThreshold); + bool notImplementedBelowThreshold = (this.NotImplemented == 0) || (this.NotImplemented < notImplementedThreshold); // Pass if below the thresholds, and sender and receiver got same amount of successess (or receiver has no results) - return statusCodeZeroBelowThreshold && unauthorizedBelowThreshold && deviceNotFoundBelowThreshold && transientErrorBelowThreshold && senderAndReceiverSuccessesPass; + return statusCodeZeroBelowThreshold && unauthorizedBelowThreshold && deviceNotFoundBelowThreshold && transientErrorBelowThreshold && senderAndReceiverSuccessesPass && notImplementedBelowThreshold; } } } diff --git a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReportGenerator.cs b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReportGenerator.cs index 616244432cc..d2268add750 100644 --- a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReportGenerator.cs +++ b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReportGenerator.cs @@ -66,6 +66,7 @@ public async Task CreateReportAsync() long deviceNotFound = 0; long transientError = 0; long resourceError = 0; + long notImplemented = 0; long receiverSuccesses = 0; Dictionary other = new Dictionary(); while (await this.SenderTestResults.MoveNextAsync()) @@ -93,6 +94,9 @@ public async Task CreateReportAsync() case 503: resourceError++; break; + case 501: + notImplemented++; + break; default: if (other.ContainsKey(statusCode)) { @@ -132,6 +136,7 @@ public async Task CreateReportAsync() deviceNotFound, transientError, resourceError, + notImplemented, other); } From f1f6ab04053c103978273ed556a1dd602782e7de Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Mon, 6 Dec 2021 18:25:53 +0000 Subject: [PATCH 7/8] bump devicenotfound tolerance for single node --- .../Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs index 4ea23b03bee..88e097ce93c 100644 --- a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs +++ b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs @@ -105,7 +105,7 @@ bool IsPassedHelper() if (!this.MqttBrokerEnabled && this.Topology == Topology.SingleNode) { - deviceNotFoundThreshold = (double)allStatusCount / 400; + deviceNotFoundThreshold = (double)allStatusCount / 200; } else if (!this.MqttBrokerEnabled && this.Topology == Topology.Nested && this.TestDescription.Contains("mqtt")) { From cadfd3db2c9246f12924d7bbade662af520eaeb1 Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Tue, 7 Dec 2021 01:06:57 +0000 Subject: [PATCH 8/8] adjust tolerances and self review --- .../Reports/CountingReportGeneratorTest.cs | 12 +++++++ .../Reports/CountingReportTest.cs | 6 ++++ .../TwinCountingReportGeneratorTest.cs | 2 ++ .../TestReportUtilTest.cs | 1 + .../Reports/CountingReport.cs | 14 +++++--- .../Reports/CountingReportGenerator.cs | 5 +++ .../LongHaul/DirectMethodLongHaulReport.cs | 29 ++++++++++++++--- .../LegacyTwin/LegacyTwinReportGenerator.cs | 32 ++++++++----------- .../Reports/TestReportGeneratorFactory.cs | 1 + 9 files changed, 74 insertions(+), 28 deletions(-) diff --git a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs index 912d384a0d2..26b5b093d4f 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs @@ -63,6 +63,7 @@ public void TestConstructorSuccess() var reportGenerator = new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), expectedSource, @@ -96,6 +97,7 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes () => new CountingReportGenerator( testDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", @@ -123,6 +125,7 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId) () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, trackingId, "expectedSource", @@ -150,6 +153,7 @@ public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expected () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), expectedSource, @@ -174,6 +178,7 @@ public void TestConstructorThrowsWhenExpectedStoreIsNotProvided() () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", @@ -201,6 +206,7 @@ public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSour () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", @@ -225,6 +231,7 @@ public void TestConstructorThrowsWhenActualStoreIsNotProvided() () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", @@ -252,6 +259,7 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType) () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", @@ -277,6 +285,7 @@ public void TestConstructorThrowsWhenTestResultComparerIsNotProvided() () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", @@ -303,6 +312,7 @@ public void TestConstructorThrowsWhenUnmatchedResultsMaxSizeIsNonPositive(ushort () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", @@ -331,6 +341,7 @@ public async Task TestCreateReportAsyncWithEmptyResults() var reportGenerator = new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), expectedSource, @@ -388,6 +399,7 @@ public async Task TestCreateReportAsync( var reportGenerator = new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), expectedSource, diff --git a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs index d6a8632b58a..5aa0d6ac61e 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs @@ -21,6 +21,7 @@ public void TestConstructorSuccess() var report = new CountingReport( TestDescription, TestMode.Connectivity, + Topology.SingleNode, false, "trackingId123", "expectedSource", @@ -78,6 +79,7 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes () => new CountingReport( testDescription, TestMode.Connectivity, + Topology.SingleNode, false, "trackingId123", "expectedSource", @@ -111,6 +113,7 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId) () => new CountingReport( TestDescription, TestMode.Connectivity, + Topology.SingleNode, false, trackingId, "expectedSource", @@ -144,6 +147,7 @@ public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expected () => new CountingReport( TestDescription, TestMode.Connectivity, + Topology.SingleNode, false, "trackingId-23434", expectedSource, @@ -177,6 +181,7 @@ public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSour () => new CountingReport( TestDescription, TestMode.Connectivity, + Topology.SingleNode, false, "trackingId-23434", "expectedSource", @@ -210,6 +215,7 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType) () => new CountingReport( TestDescription, TestMode.Connectivity, + Topology.SingleNode, false, "trackingId-23434", "expectedSource", diff --git a/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs b/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs index 14d5d23b1c3..9fb28b7cd13 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs @@ -194,6 +194,7 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType) () => new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), "expectedSource", @@ -267,6 +268,7 @@ public async Task TestCreateReportAsyncWithEmptyResults() var reportGenerator = new CountingReportGenerator( TestDescription, TestMode.Connectivity, + Topology.SingleNode, brokerEnabled, Guid.NewGuid().ToString(), expectedSource, diff --git a/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs b/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs index d6713fd6de5..4619be532c3 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs @@ -543,6 +543,7 @@ private Task MockTestResultReport(bool throwException) return Task.FromResult(new CountingReport( "mock", TestMode.Connectivity, + Topology.SingleNode, false, "mock", "mock", diff --git a/test/modules/TestResultCoordinator/Reports/CountingReport.cs b/test/modules/TestResultCoordinator/Reports/CountingReport.cs index 7b033196e29..6497e454b22 100644 --- a/test/modules/TestResultCoordinator/Reports/CountingReport.cs +++ b/test/modules/TestResultCoordinator/Reports/CountingReport.cs @@ -35,6 +35,7 @@ class CountingReport : TestResultReportBase public CountingReport( string testDescription, TestMode testMode, + Topology topology, bool mqttBrokerEnabled, string trackingId, string expectedSource, @@ -54,6 +55,7 @@ public CountingReport( : base(testDescription, trackingId, resultType) { this.TestMode = testMode; + this.Topology = topology; this.MqttBrokerEnabled = mqttBrokerEnabled; this.ExpectedSource = Preconditions.CheckNonWhiteSpace(expectedSource, nameof(expectedSource)); this.ActualSource = Preconditions.CheckNonWhiteSpace(actualSource, nameof(actualSource)); @@ -74,6 +76,8 @@ public CountingReport( [JsonConverter(typeof(StringEnumConverter))] public TestMode TestMode { get; } + public Topology Topology { get; } + public bool MqttBrokerEnabled { get; } public string ExpectedSource { get; } @@ -125,12 +129,12 @@ bool IsPassedHelper() if (this.TestMode == TestMode.Connectivity) { // Product issue for C2D messages connected to edgehub. - // We should remove this failure tolerance when fixed. if (this.TestDescription.Contains(C2dTestDescription)) { return ((double)this.TotalMatchCount / this.TotalExpectCount) > .8d; } - else if (this.TestDescription == GenericMqttTelemetryTestDescription) + // Product issue for custom mqtt telemetry. + else if (this.Topology == Topology.Nested && this.TestDescription == GenericMqttTelemetryTestDescription) { return ((double)this.TotalMatchCount / this.TotalExpectCount) > .9d; } @@ -141,11 +145,13 @@ bool IsPassedHelper() } else { - if (this.TestDescription.Contains(GenericMqttTelemetryTestDescription)) + // Product issue for custom mqtt telemetry. + if (this.Topology == Topology.Nested && this.MqttBrokerEnabled && this.TestDescription.Contains(GenericMqttTelemetryTestDescription)) { return ((double)this.TotalMatchCount / this.TotalExpectCount) > .8d; } - else if (this.MqttBrokerEnabled && this.TestDescription.Contains(MessagesTestDescription)) + // Product issue for messages when broker is enabled. + else if (this.Topology == Topology.Nested && this.MqttBrokerEnabled && this.TestDescription.Contains(MessagesTestDescription)) { return ((double)this.TotalMatchCount / this.TotalExpectCount) > .99d; } diff --git a/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs b/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs index ec99f1e9541..1aa82be08bb 100644 --- a/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs +++ b/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs @@ -23,6 +23,7 @@ sealed class CountingReportGenerator : ITestResultReportGenerator internal CountingReportGenerator( string testDescription, TestMode testMode, + Topology topology, bool mqttBrokerEnabled, string trackingId, string expectedSource, @@ -36,6 +37,7 @@ internal CountingReportGenerator( { this.TestDescription = Preconditions.CheckNonWhiteSpace(testDescription, nameof(testDescription)); this.TestMode = testMode; + this.Topology = topology; this.MqttBrokerEnabled = mqttBrokerEnabled; this.trackingId = Preconditions.CheckNonWhiteSpace(trackingId, nameof(trackingId)); this.ExpectedTestResults = Preconditions.CheckNotNull(expectedTestResults, nameof(expectedTestResults)); @@ -50,6 +52,8 @@ internal CountingReportGenerator( internal TestMode TestMode { get; } + internal Topology Topology { get; } + internal bool MqttBrokerEnabled { get; } internal string ActualSource { get; } @@ -240,6 +244,7 @@ public async Task CreateReportAsync() return new CountingReport( this.TestDescription, this.TestMode, + this.Topology, this.MqttBrokerEnabled, this.trackingId, this.ExpectedSource, diff --git a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs index 88e097ce93c..2477f2268f3 100644 --- a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs +++ b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. namespace TestResultCoordinator.Reports.DirectMethod.LongHaul { + using System; using System.Collections.Generic; using System.Linq; using System.Net; @@ -84,7 +85,7 @@ bool IsPassedHelper() } bool senderAndReceiverSuccessesPass = this.SenderSuccesses <= this.ReceiverSuccesses; - long allStatusCount = this.SenderSuccesses + this.StatusCodeZero + this.Other.Sum(x => x.Value); + long allStatusCount = this.SenderSuccesses + this.StatusCodeZero + this.Unauthorized + this.DeviceNotFound + this.TransientError + this.ResourceError + this.NotImplemented + this.Other.Sum(x => x.Value); double statusCodeZeroThreshold; double unauthorizedThreshold; @@ -98,16 +99,34 @@ bool IsPassedHelper() // We expect to get this status sometimes because of edgehub restarts, but if we receive too many we should fail the tests. // TODO: When the SDK allows edgehub to de-register from subscriptions and we make the fix in edgehub, then we can fail tests for any status code 0. statusCodeZeroThreshold = (double)allStatusCount / 1000; - unauthorizedThreshold = (double)allStatusCount / 1000; + + // Sometimes transient network/resource errors are caught necessitating a tolerance. transientErrorThreshold = (double)allStatusCount / 1000; resourceErrorThreshold = (double)allStatusCount / 1000; - notImplementedThreshold = (double)allStatusCount / 1000; - if (!this.MqttBrokerEnabled && this.Topology == Topology.SingleNode) + // Sometimes iothub returns Unauthorized or NotImplemented that then later recovers. + // Only occurs with broker enabled, so only apply tolerance in this case. + if (this.MqttBrokerEnabled) + { + unauthorizedThreshold = (double)allStatusCount / 1000; + notImplementedThreshold = (double)allStatusCount / 1000; + } + else + { + unauthorizedThreshold = (double)allStatusCount / double.MaxValue; + notImplementedThreshold = (double)allStatusCount / double.MaxValue; + } + + // DeviceNotFound typically happens when EdgeHub restarts and is offline. + // For different test suites this happens at different rates. + // 1) Single node runs arm devices, so this tolerance is a bit lenient. + // 2) Nested non-broker has some product issue where we need some tolerance. + // 3) Nested broker-enabled is the most stable. + if (this.Topology == Topology.SingleNode && !this.MqttBrokerEnabled) { deviceNotFoundThreshold = (double)allStatusCount / 200; } - else if (!this.MqttBrokerEnabled && this.Topology == Topology.Nested && this.TestDescription.Contains("mqtt")) + else if (this.Topology == Topology.Nested && !this.MqttBrokerEnabled) { deviceNotFoundThreshold = (double)allStatusCount / 250; } diff --git a/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs b/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs index 8eb7c101816..dd4f271f4a9 100644 --- a/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs +++ b/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs @@ -77,34 +77,28 @@ public async Task CreateReportAsync() } // See TwinTester/StatusCode.cs for reference. - bool IsPassed(IDictionary results) + bool IsPassed(IDictionary statusCodesToCount) { bool isPassed = true; - int totalResults = results.Sum(x => x.Value); + int totalResults = statusCodesToCount.Sum(x => x.Value); if (totalResults == 0) { return false; } - if (this.Topology == Topology.Nested) + // Product issue where: + // 1) We don't receive some desired properties in module-registered twin desired property callback + // 1) Module cannot make reported property update + if (this.Topology == Topology.Nested && this.MqttBrokerEnabled) { - if (this.MqttBrokerEnabled || this.TestDescription.Contains("amqp")) - { - int[] bigToleranceStatusCodes = { }; - int[] littleToleranceStatusCodes = { 501, 504 }; - isPassed = this.GeneratePassResult(results, bigToleranceStatusCodes, littleToleranceStatusCodes); - } - else - { - int[] bigToleranceStatusCodes = { 505, 503, 506 }; - int[] littleToleranceStatusCodes = { 501, 502, 504 }; - isPassed = this.GeneratePassResult(results, bigToleranceStatusCodes, littleToleranceStatusCodes); - } + int[] bigToleranceStatusCodes = { }; + int[] littleToleranceStatusCodes = { 501, 504 }; + isPassed = this.GeneratePassResult(statusCodesToCount, bigToleranceStatusCodes, littleToleranceStatusCodes); } else { - List statusCodes = results.Keys.ToList(); + List statusCodes = statusCodesToCount.Keys.ToList(); IEnumerable failingStatusCodes = statusCodes.Where(s => { string statusCode = s.ToString(); @@ -117,10 +111,10 @@ bool IsPassed(IDictionary results) return isPassed; } - bool GeneratePassResult(IDictionary results, int[] bigToleranceStatusCodes, int[] littleToleranceStatusCodes) + bool GeneratePassResult(IDictionary statusCodesToCount, int[] bigToleranceStatusCodes, int[] littleToleranceStatusCodes) { - int totalResults = results.Sum(x => x.Value); - foreach (KeyValuePair statusCodeToCount in results) + int totalResults = statusCodesToCount.Sum(x => x.Value); + foreach (KeyValuePair statusCodeToCount in statusCodesToCount) { int statusCode = statusCodeToCount.Key; int statusCodeCount = statusCodeToCount.Value; diff --git a/test/modules/TestResultCoordinator/Reports/TestReportGeneratorFactory.cs b/test/modules/TestResultCoordinator/Reports/TestReportGeneratorFactory.cs index c0ca8bc9af7..32f93513b9c 100644 --- a/test/modules/TestResultCoordinator/Reports/TestReportGeneratorFactory.cs +++ b/test/modules/TestResultCoordinator/Reports/TestReportGeneratorFactory.cs @@ -56,6 +56,7 @@ await this.LonghaulSettings.ForEachAsync(async (longhaulSettings) => return new CountingReportGenerator( metadata.TestDescription, Settings.Current.TestMode, + Settings.Current.Topology, Settings.Current.MqttBrokerEnabled, trackingId, metadata.ExpectedSource,