diff --git a/test/modules/TestResultCoordinator/Reports/CountingReport.cs b/test/modules/TestResultCoordinator/Reports/CountingReport.cs index d15c5e5973c..6497e454b22 100644 --- a/test/modules/TestResultCoordinator/Reports/CountingReport.cs +++ b/test/modules/TestResultCoordinator/Reports/CountingReport.cs @@ -126,19 +126,39 @@ bool IsPassedHelper() }, () => { - // Product issue for C2D messages connected to edgehub over mqtt. - // We should remove this failure tolerance when fixed. - if (this.TestDescription.Contains(C2dTestDescription)) + if (this.TestMode == TestMode.Connectivity) { - return ((double)this.TotalMatchCount / this.TotalExpectCount) > .8d; - } - else if (this.TestDescription == GenericMqttTelemetryTestDescription && this.TestMode == TestMode.Connectivity) - { - return ((double)this.TotalMatchCount / this.TotalExpectCount) > .9d; + // Product issue for C2D messages connected to edgehub. + if (this.TestDescription.Contains(C2dTestDescription)) + { + return ((double)this.TotalMatchCount / this.TotalExpectCount) > .8d; + } + // Product issue for custom mqtt telemetry. + else if (this.Topology == Topology.Nested && this.TestDescription == GenericMqttTelemetryTestDescription) + { + return ((double)this.TotalMatchCount / this.TotalExpectCount) > .9d; + } + else + { + return this.TotalExpectCount == this.TotalMatchCount; + } } else { - return this.TotalExpectCount == this.TotalMatchCount; + // Product issue for custom mqtt telemetry. + if (this.Topology == Topology.Nested && this.MqttBrokerEnabled && this.TestDescription.Contains(GenericMqttTelemetryTestDescription)) + { + return ((double)this.TotalMatchCount / this.TotalExpectCount) > .8d; + } + // Product issue for messages when broker is enabled. + else if (this.Topology == Topology.Nested && this.MqttBrokerEnabled && this.TestDescription.Contains(MessagesTestDescription)) + { + return ((double)this.TotalMatchCount / this.TotalExpectCount) > .99d; + } + else + { + return this.TotalExpectCount == this.TotalMatchCount; + } } }); } diff --git a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs index 4fe3fa830dd..2477f2268f3 100644 --- a/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs +++ b/test/modules/TestResultCoordinator/Reports/DirectMethod/LongHaul/DirectMethodLongHaulReport.cs @@ -85,20 +85,65 @@ bool IsPassedHelper() } bool senderAndReceiverSuccessesPass = this.SenderSuccesses <= this.ReceiverSuccesses; + long allStatusCount = this.SenderSuccesses + this.StatusCodeZero + this.Unauthorized + this.DeviceNotFound + this.TransientError + this.ResourceError + this.NotImplemented + this.Other.Sum(x => x.Value); + + double statusCodeZeroThreshold; + double unauthorizedThreshold; + double deviceNotFoundThreshold; + double transientErrorThreshold; + double resourceErrorThreshold; + double notImplementedThreshold; // The SDK does not allow edgehub to de-register from iothub subscriptions, which results in DirectMethod clients sometimes receiving status code 0. // Github issue: https://github.com/Azure/iotedge/issues/681 // We expect to get this status sometimes because of edgehub restarts, but if we receive too many we should fail the tests. // TODO: When the SDK allows edgehub to de-register from subscriptions and we make the fix in edgehub, then we can fail tests for any status code 0. - long allStatusCount = this.SenderSuccesses + this.StatusCodeZero + this.Other.Sum(x => x.Value); - bool statusCodeZeroBelowThreshold = (this.StatusCodeZero == 0) || (this.StatusCodeZero < ((double)allStatusCount / 1000)); - bool unauthorizedBelowThreshold = (this.Unauthorized == 0) || (this.Unauthorized < ((double)allStatusCount / 1000)); - bool deviceNotFoundBelowThreshold = (this.DeviceNotFound == 0) || (this.DeviceNotFound < ((double)allStatusCount / 100)); - bool transientErrorBelowThreshold = (this.TransientError == 0) || (this.TransientError < ((double)allStatusCount / 100)); - bool resourceErrorBelowThreshold = (this.ResourceError == 0) || (this.ResourceError < ((double)allStatusCount / 100)); + statusCodeZeroThreshold = (double)allStatusCount / 1000; + + // Sometimes transient network/resource errors are caught necessitating a tolerance. + transientErrorThreshold = (double)allStatusCount / 1000; + resourceErrorThreshold = (double)allStatusCount / 1000; + + // Sometimes iothub returns Unauthorized or NotImplemented that then later recovers. + // Only occurs with broker enabled, so only apply tolerance in this case. + if (this.MqttBrokerEnabled) + { + unauthorizedThreshold = (double)allStatusCount / 1000; + notImplementedThreshold = (double)allStatusCount / 1000; + } + else + { + unauthorizedThreshold = (double)allStatusCount / double.MaxValue; + notImplementedThreshold = (double)allStatusCount / double.MaxValue; + } + + // DeviceNotFound typically happens when EdgeHub restarts and is offline. + // For different test suites this happens at different rates. + // 1) Single node runs arm devices, so this tolerance is a bit lenient. + // 2) Nested non-broker has some product issue where we need some tolerance. + // 3) Nested broker-enabled is the most stable. + if (this.Topology == Topology.SingleNode && !this.MqttBrokerEnabled) + { + deviceNotFoundThreshold = (double)allStatusCount / 200; + } + else if (this.Topology == Topology.Nested && !this.MqttBrokerEnabled) + { + deviceNotFoundThreshold = (double)allStatusCount / 250; + } + else + { + deviceNotFoundThreshold = (double)allStatusCount / 100; + } + + bool statusCodeZeroBelowThreshold = (this.StatusCodeZero == 0) || (this.StatusCodeZero < statusCodeZeroThreshold); + bool unauthorizedBelowThreshold = (this.Unauthorized == 0) || (this.Unauthorized < unauthorizedThreshold); + bool deviceNotFoundBelowThreshold = (this.DeviceNotFound == 0) || (this.DeviceNotFound < deviceNotFoundThreshold); + bool transientErrorBelowThreshold = (this.TransientError == 0) || (this.TransientError < transientErrorThreshold); + bool resourceErrorBelowThreshold = (this.ResourceError == 0) || (this.ResourceError < resourceErrorThreshold); + bool notImplementedBelowThreshold = (this.NotImplemented == 0) || (this.NotImplemented < notImplementedThreshold); // Pass if below the thresholds, and sender and receiver got same amount of successess (or receiver has no results) - return statusCodeZeroBelowThreshold && unauthorizedBelowThreshold && deviceNotFoundBelowThreshold && transientErrorBelowThreshold && senderAndReceiverSuccessesPass; + return statusCodeZeroBelowThreshold && unauthorizedBelowThreshold && deviceNotFoundBelowThreshold && transientErrorBelowThreshold && senderAndReceiverSuccessesPass && notImplementedBelowThreshold; } } } diff --git a/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs b/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs index c971ab093f5..dd4f271f4a9 100644 --- a/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs +++ b/test/modules/TestResultCoordinator/Reports/LegacyTwin/LegacyTwinReportGenerator.cs @@ -11,6 +11,8 @@ namespace TestResultCoordinator.Reports.LegacyTwin sealed class LegacyTwinReportGenerator : ITestResultReportGenerator { + const double BigToleranceProportion = .005; + const double LittleToleranceProportion = .001; static readonly ILogger Logger = ModuleUtil.CreateLogger(nameof(LegacyTwinReportGenerator)); readonly string trackingId; @@ -47,15 +49,9 @@ public async Task CreateReportAsync() { Logger.LogInformation($"Start to generate report by {nameof(LegacyTwinReportGenerator)} for Sources [{this.SenderSource}] "); IDictionary results = new Dictionary(); - bool isPassed = true; while (await this.SenderTestResults.MoveNextAsync()) { int status = int.Parse(this.SenderTestResults.Current.Result.Substring(0, 3)); - if (status > 299) - { - isPassed = false; - } - if (results.ContainsKey(status)) { results[status] = results[status] + 1; @@ -66,6 +62,8 @@ public async Task CreateReportAsync() } } + bool isPassed = this.IsPassed(results); + var report = new LegacyTwinReport( this.TestDescription, this.trackingId, @@ -77,5 +75,76 @@ public async Task CreateReportAsync() Logger.LogInformation($"Successfully finished creating LegacyTwinReport for Source [{this.SenderSource}]"); return report; } + + // See TwinTester/StatusCode.cs for reference. + bool IsPassed(IDictionary statusCodesToCount) + { + bool isPassed = true; + int totalResults = statusCodesToCount.Sum(x => x.Value); + + if (totalResults == 0) + { + return false; + } + + // Product issue where: + // 1) We don't receive some desired properties in module-registered twin desired property callback + // 1) Module cannot make reported property update + if (this.Topology == Topology.Nested && this.MqttBrokerEnabled) + { + int[] bigToleranceStatusCodes = { }; + int[] littleToleranceStatusCodes = { 501, 504 }; + isPassed = this.GeneratePassResult(statusCodesToCount, bigToleranceStatusCodes, littleToleranceStatusCodes); + } + else + { + List statusCodes = statusCodesToCount.Keys.ToList(); + IEnumerable failingStatusCodes = statusCodes.Where(s => + { + string statusCode = s.ToString(); + return !statusCode.StartsWith("2"); + }); + + isPassed = failingStatusCodes.Count() == 0; + } + + return isPassed; + } + + bool GeneratePassResult(IDictionary statusCodesToCount, int[] bigToleranceStatusCodes, int[] littleToleranceStatusCodes) + { + int totalResults = statusCodesToCount.Sum(x => x.Value); + foreach (KeyValuePair statusCodeToCount in statusCodesToCount) + { + int statusCode = statusCodeToCount.Key; + int statusCodeCount = statusCodeToCount.Value; + + // ignore the status codes indicating some success + if (statusCode.ToString().StartsWith("2")) + { + continue; + } + else if (bigToleranceStatusCodes.Contains(statusCode)) + { + if ((double)statusCodeCount / totalResults > BigToleranceProportion) + { + return false; + } + } + else if (littleToleranceStatusCodes.Contains(statusCode)) + { + if ((double)statusCodeCount / totalResults > LittleToleranceProportion) + { + return false; + } + } + else + { + return false; + } + } + + return true; + } } }