diff --git a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParserStateObject.cs b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParserStateObject.cs index 4fedec53ac..3fc29c3d52 100644 --- a/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParserStateObject.cs +++ b/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParserStateObject.cs @@ -125,8 +125,14 @@ internal enum SnapshottedStateFlags : byte internal volatile bool _attentionSent; // true if we sent an Attention to the server internal volatile bool _attentionSending; - private readonly LastIOTimer _lastSuccessfulIOTimer; - + // Below 2 properties are used to enforce timeout delays in code to + // reproduce issues related to theadpool starvation and timeout delay. + // It should always be set to false by default, and only be enabled during testing. + internal bool _enforceTimeoutDelay = false; + internal int _enforcedTimeoutDelayInMilliSeconds = 5000; + + private readonly LastIOTimer _lastSuccessfulIOTimer; + // secure password information to be stored // At maximum number of secure string that need to be stored is two; one for login password and the other for new change password private SecureString[] _securePasswords = new SecureString[2] { null, null }; @@ -1455,7 +1461,7 @@ internal bool TryReadInt16(out short value) { // The entire int16 is in the packet and in the buffer, so just return it // and take care of the counters. - buffer = _inBuff.AsSpan(_inBytesUsed,2); + buffer = _inBuff.AsSpan(_inBytesUsed, 2); _inBytesUsed += 2; _inBytesPacket -= 2; } @@ -1489,7 +1495,7 @@ internal bool TryReadInt32(out int value) } AssertValidState(); - value = (buffer[3] << 24) + (buffer[2] <<16) + (buffer[1] << 8) + buffer[0]; + value = (buffer[3] << 24) + (buffer[2] << 16) + (buffer[1] << 8) + buffer[0]; return true; } @@ -2277,9 +2283,11 @@ private sealed class TimeoutState private void OnTimeoutAsync(object state) { -#if DEBUG - Thread.Sleep(13000); -#endif + if (_enforceTimeoutDelay) + { + Thread.Sleep(_enforcedTimeoutDelayInMilliSeconds); + } + int currentIdentityValue = _timeoutIdentityValue; TimeoutState timeoutState = (TimeoutState)state; if (timeoutState.IdentityValue == _timeoutIdentityValue) @@ -2467,7 +2475,7 @@ internal void ReadSni(TaskCompletionSource completion) Timeout.Infinite, Timeout.Infinite ); - + // -1 == Infinite // 0 == Already timed out (NOTE: To simulate the same behavior as sync we will only timeout on 0 if we receive an IO Pending from SNI) @@ -3555,11 +3563,10 @@ internal void SendAttention(bool mustTakeWriteLock = false) // Set _attentionSending to true before sending attention and reset after setting _attentionSent // This prevents a race condition between receiving the attention ACK and setting _attentionSent _attentionSending = true; - #if DEBUG if (!_skipSendAttention) - { #endif + { // Take lock and send attention bool releaseLock = false; if ((mustTakeWriteLock) && (!_parser.Connection.ThreadHasParserLockForClose)) @@ -3589,9 +3596,7 @@ internal void SendAttention(bool mustTakeWriteLock = false) _parser.Connection._parserLock.Release(); } } -#if DEBUG } -#endif SetTimeoutSeconds(AttentionTimeoutSeconds); // Initialize new attention timeout of 5 seconds. _attentionSent = true; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj index 2a5bf658c5..45fc752361 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj @@ -66,6 +66,7 @@ + diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AsyncTest/AsyncTimeoutTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AsyncTest/AsyncTimeoutTest.cs new file mode 100644 index 0000000000..0ba98d83b6 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AsyncTest/AsyncTimeoutTest.cs @@ -0,0 +1,209 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Data; +using System.Threading.Tasks; +using System.Xml; +using Microsoft.Data.SqlClient.ManualTesting.Tests.SystemDataInternals; +using Xunit; + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests +{ + public static class AsyncTimeoutTest + { + static string delayQuery2s = "WAITFOR DELAY '00:00:02'"; + static string delayQuery10s = "WAITFOR DELAY '00:00:10'"; + + public enum AsyncAPI + { + ExecuteReaderAsync, + ExecuteScalarAsync, + ExecuteXmlReaderAsync + } + + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] + [ClassData(typeof(AsyncTimeoutTestVariations))] + public static void TestDelayedAsyncTimeout(AsyncAPI api, string commonObj, int delayPeriod, bool marsEnabled) => + RunTest(api, commonObj, delayPeriod, marsEnabled); + + public class AsyncTimeoutTestVariations : IEnumerable + { + public IEnumerator GetEnumerator() + { + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 8000, true }; + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 5000, true }; + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 0, true }; + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 8000, false }; + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 5000, false }; + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Connection", 0, false }; + + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 8000, true }; + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 5000, true }; + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 0, true }; + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 8000, false }; + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 5000, false }; + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Connection", 0, false }; + + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 8000, true }; + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 5000, true }; + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 0, true }; + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 8000, false }; + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 5000, false }; + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Connection", 0, false }; + + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 8000, true }; + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 5000, true }; + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 0, true }; + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 8000, false }; + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 5000, false }; + yield return new object[] { AsyncAPI.ExecuteReaderAsync, "Command", 0, false }; + + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 8000, true }; + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 5000, true }; + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 0, true }; + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 8000, false }; + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 5000, false }; + yield return new object[] { AsyncAPI.ExecuteScalarAsync, "Command", 0, false }; + + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 8000, true }; + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 5000, true }; + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 0, true }; + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 8000, false }; + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 5000, false }; + yield return new object[] { AsyncAPI.ExecuteXmlReaderAsync, "Command", 0, false }; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + private static void RunTest(AsyncAPI api, string commonObj, int timeoutDelay, bool marsEnabled) + { + string connString = new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString) + { + MultipleActiveResultSets = marsEnabled + }.ConnectionString; + + using (SqlConnection sqlConnection = new SqlConnection(connString)) + { + sqlConnection.Open(); + if (timeoutDelay != 0) + { + ConnectionHelper.SetEnforcedTimeout(sqlConnection, true, timeoutDelay); + } + switch (commonObj) + { + case "Connection": + QueryAndValidate(api, 1, delayQuery2s, 1, true, true, sqlConnection).Wait(); + QueryAndValidate(api, 2, delayQuery2s, 5, false, true, sqlConnection).Wait(); + QueryAndValidate(api, 3, delayQuery10s, 1, true, true, sqlConnection).Wait(); + QueryAndValidate(api, 4, delayQuery2s, 10, false, true, sqlConnection).Wait(); + break; + case "Command": + using (SqlCommand cmd = sqlConnection.CreateCommand()) + { + QueryAndValidate(api, 1, delayQuery2s, 1, true, false, sqlConnection, cmd).Wait(); + QueryAndValidate(api, 2, delayQuery2s, 5, false, false, sqlConnection, cmd).Wait(); + QueryAndValidate(api, 3, delayQuery10s, 1, true, false, sqlConnection, cmd).Wait(); + QueryAndValidate(api, 4, delayQuery2s, 10, false, false, sqlConnection, cmd).Wait(); + } + break; + } + } + } + + private static async Task QueryAndValidate(AsyncAPI api, int index, string delayQuery, int timeout, + bool timeoutExExpected = false, bool useTransaction = false, SqlConnection cn = null, SqlCommand cmd = null) + { + SqlTransaction tx = null; + try + { + if (cn != null) + { + if (cn.State != ConnectionState.Open) + { + await cn.OpenAsync(); + } + cmd = cn.CreateCommand(); + if (useTransaction) + { + tx = cn.BeginTransaction(IsolationLevel.ReadCommitted); + cmd.Transaction = tx; + } + } + + cmd.CommandTimeout = timeout; + if (api != AsyncAPI.ExecuteXmlReaderAsync) + { + cmd.CommandText = delayQuery + $";select {index} as Id;"; + } + else + { + cmd.CommandText = delayQuery + $";select {index} as Id FOR XML PATH;"; + } + + var result = -1; + switch (api) + { + case AsyncAPI.ExecuteReaderAsync: + using (SqlDataReader reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) + { + while (await reader.ReadAsync().ConfigureAwait(false)) + { + var columnIndex = reader.GetOrdinal("Id"); + result = reader.GetInt32(columnIndex); + break; + } + } + break; + case AsyncAPI.ExecuteScalarAsync: + result = (int)await cmd.ExecuteScalarAsync().ConfigureAwait(false); + break; + case AsyncAPI.ExecuteXmlReaderAsync: + using (XmlReader reader = await cmd.ExecuteXmlReaderAsync().ConfigureAwait(false)) + { + try + { + Assert.True(reader.Settings.Async); + reader.ReadToDescendant("Id"); + result = reader.ReadElementContentAsInt(); + } + catch (Exception ex) + { + Assert.False(true, "Exception occurred: " + ex.Message); + } + } + break; + } + + if (result != index) + { + throw new Exception("High Alert! Wrong data received for index: " + index); + } + else + { + Assert.True(!timeoutExExpected && result == index); + } + } + catch (SqlException e) + { + if (!timeoutExExpected) + throw new Exception("Index " + index + " failed with: " + e.Message); + else + Assert.True(timeoutExExpected && e.Class == 11 && e.Number == -2); + } + finally + { + if (cn != null) + { + if (useTransaction) + tx.Commit(); + cn.Close(); + } + } + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/Common/SystemDataInternals/ConnectionHelper.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/Common/SystemDataInternals/ConnectionHelper.cs index 2b4f533dd5..54561e1be9 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/Common/SystemDataInternals/ConnectionHelper.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/Common/SystemDataInternals/ConnectionHelper.cs @@ -10,15 +10,22 @@ namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SystemDataInternals { internal static class ConnectionHelper { - private static Assembly s_systemDotData = Assembly.Load(new AssemblyName(typeof(SqlConnection).GetTypeInfo().Assembly.FullName)); - private static Type s_sqlConnection = s_systemDotData.GetType("Microsoft.Data.SqlClient.SqlConnection"); - private static Type s_sqlInternalConnection = s_systemDotData.GetType("Microsoft.Data.SqlClient.SqlInternalConnection"); - private static Type s_sqlInternalConnectionTds = s_systemDotData.GetType("Microsoft.Data.SqlClient.SqlInternalConnectionTds"); - private static Type s_dbConnectionInternal = s_systemDotData.GetType("Microsoft.Data.ProviderBase.DbConnectionInternal"); + private static Assembly s_MicrosoftDotData = Assembly.Load(new AssemblyName(typeof(SqlConnection).GetTypeInfo().Assembly.FullName)); + private static Type s_sqlConnection = s_MicrosoftDotData.GetType("Microsoft.Data.SqlClient.SqlConnection"); + private static Type s_sqlInternalConnection = s_MicrosoftDotData.GetType("Microsoft.Data.SqlClient.SqlInternalConnection"); + private static Type s_sqlInternalConnectionTds = s_MicrosoftDotData.GetType("Microsoft.Data.SqlClient.SqlInternalConnectionTds"); + private static Type s_dbConnectionInternal = s_MicrosoftDotData.GetType("Microsoft.Data.ProviderBase.DbConnectionInternal"); + private static Type s_tdsParser = s_MicrosoftDotData.GetType("Microsoft.Data.SqlClient.TdsParser"); + private static Type s_tdsParserStateObject = s_MicrosoftDotData.GetType("Microsoft.Data.SqlClient.TdsParserStateObject"); private static PropertyInfo s_sqlConnectionInternalConnection = s_sqlConnection.GetProperty("InnerConnection", BindingFlags.Instance | BindingFlags.NonPublic); private static PropertyInfo s_dbConnectionInternalPool = s_dbConnectionInternal.GetProperty("Pool", BindingFlags.Instance | BindingFlags.NonPublic); private static MethodInfo s_dbConnectionInternalIsConnectionAlive = s_dbConnectionInternal.GetMethod("IsConnectionAlive", BindingFlags.Instance | BindingFlags.NonPublic); private static FieldInfo s_sqlInternalConnectionTdsParser = s_sqlInternalConnectionTds.GetField("_parser", BindingFlags.Instance | BindingFlags.NonPublic); + private static PropertyInfo s_innerConnectionProperty = s_sqlConnection.GetProperty("InnerConnection", BindingFlags.Instance | BindingFlags.NonPublic); + private static PropertyInfo s_tdsParserProperty = s_sqlInternalConnectionTds.GetProperty("Parser", BindingFlags.Instance | BindingFlags.NonPublic); + private static FieldInfo s_tdsParserStateObjectProperty = s_tdsParser.GetField("_physicalStateObj", BindingFlags.Instance | BindingFlags.NonPublic); + private static FieldInfo s_enforceTimeoutDelayProperty = s_tdsParserStateObject.GetField("_enforceTimeoutDelay", BindingFlags.Instance | BindingFlags.NonPublic); + private static FieldInfo s_enforcedTimeoutDelayInMilliSeconds = s_tdsParserStateObject.GetField("_enforcedTimeoutDelayInMilliSeconds", BindingFlags.Instance | BindingFlags.NonPublic); public static object GetConnectionPool(object internalConnection) { @@ -28,12 +35,12 @@ public static object GetConnectionPool(object internalConnection) public static object GetInternalConnection(this SqlConnection connection) { + VerifyObjectIsConnection(connection); object internalConnection = s_sqlConnectionInternalConnection.GetValue(connection, null); Debug.Assert(((internalConnection != null) && (s_dbConnectionInternal.IsInstanceOfType(internalConnection))), "Connection provided has an invalid internal connection"); return internalConnection; } - public static bool IsConnectionAlive(object internalConnection) { VerifyObjectIsInternalConnection(internalConnection); @@ -45,7 +52,15 @@ private static void VerifyObjectIsInternalConnection(object internalConnection) if (internalConnection == null) throw new ArgumentNullException(nameof(internalConnection)); if (!s_dbConnectionInternal.IsInstanceOfType(internalConnection)) - throw new ArgumentException("Object provided was not a DbConnectionInternal", "internalConnection"); + throw new ArgumentException("Object provided was not a DbConnectionInternal", nameof(internalConnection)); + } + + private static void VerifyObjectIsConnection(object connection) + { + if (connection == null) + throw new ArgumentNullException(nameof(connection)); + if (!s_sqlConnection.IsInstanceOfType(connection)) + throw new ArgumentException("Object provided was not a SqlConnection", nameof(connection)); } public static object GetParser(object internalConnection) @@ -53,5 +68,16 @@ public static object GetParser(object internalConnection) VerifyObjectIsInternalConnection(internalConnection); return s_sqlInternalConnectionTdsParser.GetValue(internalConnection); } + + public static void SetEnforcedTimeout(this SqlConnection connection, bool enforce, int timeout) + { + VerifyObjectIsConnection(connection); + var stateObj = s_tdsParserStateObjectProperty.GetValue( + s_tdsParserProperty.GetValue( + s_innerConnectionProperty.GetValue( + connection, null), null)); + s_enforceTimeoutDelayProperty.SetValue(stateObj, enforce); + s_enforcedTimeoutDelayInMilliSeconds.SetValue(stateObj, timeout); + } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/Common/SystemDataInternals/ConnectionPoolHelper.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/Common/SystemDataInternals/ConnectionPoolHelper.cs index 6ae73f5571..d7c5471427 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/Common/SystemDataInternals/ConnectionPoolHelper.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/Common/SystemDataInternals/ConnectionPoolHelper.cs @@ -13,13 +13,13 @@ namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SystemDataInternals { internal static class ConnectionPoolHelper { - private static Assembly s_systemDotData = Assembly.Load(new AssemblyName(typeof(SqlConnection).GetTypeInfo().Assembly.FullName)); - private static Type s_dbConnectionPool = s_systemDotData.GetType("Microsoft.Data.ProviderBase.DbConnectionPool"); - private static Type s_dbConnectionPoolGroup = s_systemDotData.GetType("Microsoft.Data.ProviderBase.DbConnectionPoolGroup"); - private static Type s_dbConnectionPoolIdentity = s_systemDotData.GetType("Microsoft.Data.ProviderBase.DbConnectionPoolIdentity"); - private static Type s_dbConnectionFactory = s_systemDotData.GetType("Microsoft.Data.ProviderBase.DbConnectionFactory"); - private static Type s_sqlConnectionFactory = s_systemDotData.GetType("Microsoft.Data.SqlClient.SqlConnectionFactory"); - private static Type s_dbConnectionPoolKey = s_systemDotData.GetType("Microsoft.Data.Common.DbConnectionPoolKey"); + private static Assembly s_MicrosoftDotData = Assembly.Load(new AssemblyName(typeof(SqlConnection).GetTypeInfo().Assembly.FullName)); + private static Type s_dbConnectionPool = s_MicrosoftDotData.GetType("Microsoft.Data.ProviderBase.DbConnectionPool"); + private static Type s_dbConnectionPoolGroup = s_MicrosoftDotData.GetType("Microsoft.Data.ProviderBase.DbConnectionPoolGroup"); + private static Type s_dbConnectionPoolIdentity = s_MicrosoftDotData.GetType("Microsoft.Data.ProviderBase.DbConnectionPoolIdentity"); + private static Type s_dbConnectionFactory = s_MicrosoftDotData.GetType("Microsoft.Data.ProviderBase.DbConnectionFactory"); + private static Type s_sqlConnectionFactory = s_MicrosoftDotData.GetType("Microsoft.Data.SqlClient.SqlConnectionFactory"); + private static Type s_dbConnectionPoolKey = s_MicrosoftDotData.GetType("Microsoft.Data.Common.DbConnectionPoolKey"); private static Type s_dictStringPoolGroup = typeof(Dictionary<,>).MakeGenericType(s_dbConnectionPoolKey, s_dbConnectionPoolGroup); private static Type s_dictPoolIdentityPool = typeof(ConcurrentDictionary<,>).MakeGenericType(s_dbConnectionPoolIdentity, s_dbConnectionPool); private static PropertyInfo s_dbConnectionPoolCount = s_dbConnectionPool.GetProperty("Count", BindingFlags.Instance | BindingFlags.NonPublic); @@ -123,7 +123,6 @@ internal static int CountConnectionsInPool(object pool) return (int)s_dbConnectionPoolCount.GetValue(pool, null); } - private static void VerifyObjectIsPool(object pool) { if (pool == null)