Add support for UTF8 Feature Extension

src/System.Data.SqlClient/src/System/Data/SqlClient/SqlInternalConnectionTds.cs

@@ -1157,8 +1157,8 @@ private void Login(ServerInfo server, TimeoutTimer timeout, string newPassword,
                 _federatedAuthenticationRequested = true;
             }
 
-            // The GLOBALTRANSACTIONS feature is implicitly requested
-            requestedFeatures |= TdsEnums.FeatureExtension.GlobalTransactions;
+            // The GLOBALTRANSACTIONS and UTF8 support features are implicitly requested
+            requestedFeatures |= TdsEnums.FeatureExtension.GlobalTransactions | TdsEnums.FeatureExtension.UTF8Support;
             _parser.TdsLogin(login, requestedFeatures, _recoverySessionData, _fedAuthFeatureExtensionData);
         }
 
@@ -1959,6 +1959,14 @@ internal void OnFeatureExtAck(int featureId, byte[] data)
                         break;
                     }
 
+                case TdsEnums.FEATUREEXT_UTF8SUPPORT:
+                    {
+                        if (data.Length < 1)
+                        {
+                            throw SQL.ParsingError();
+                        }
+                        break;
+                    }
                 default:
                     {
                         // Unknown feature ack 

src/System.Data.SqlClient/src/System/Data/SqlClient/TdsEnums.cs

@@ -201,18 +201,22 @@ public enum EnvChangeType : byte
         // Feature Extension
         public const byte FEATUREEXT_TERMINATOR = 0xFF;
         public const byte FEATUREEXT_SRECOVERY = 0x01;
-        public const byte FEATUREEXT_GLOBALTRANSACTIONS = 0x05;
         public const byte FEATUREEXT_FEDAUTH = 0x02;
+        public const byte FEATUREEXT_GLOBALTRANSACTIONS = 0x05;
+        public const byte FEATUREEXT_UTF8SUPPORT = 0x0A;
 
         [Flags]
         public enum FeatureExtension : uint
         {
             None = 0,
             SessionRecovery = 1,
             FedAuth = 2,
-            GlobalTransactions = 8,
+            GlobalTransactions = 16,
+            UTF8Support = 512,
         }
 
+        public const uint UTF8_IN_TDSCOLLATION = 0x4000000;
+
         public const byte FEDAUTHLIB_LIVEID = 0X00;
         public const byte FEDAUTHLIB_SECURITYTOKEN = 0x01;
         public const byte FEDAUTHLIB_ADAL = 0x02;

src/System.Data.SqlClient/src/System/Data/SqlClient/TdsParser.cs

@@ -2281,14 +2281,23 @@ private bool TryProcessEnvChange(int tokenLength, TdsParserStateObject stateObj,
                             {
                                 return false;
                             }
-
-                            // give the parser the new collation values in case parameters don't specify one
+                            
+                            // Give the parser the new collation values in case parameters don't specify one
                             _defaultCollation = env.newCollation;
-                            int newCodePage = GetCodePage(env.newCollation, stateObj);
-                            if (newCodePage != _defaultCodePage)
+
+                            // UTF8 collation
+                            if ((env.newCollation.info & TdsEnums.UTF8_IN_TDSCOLLATION) == TdsEnums.UTF8_IN_TDSCOLLATION)
                             {
-                                _defaultCodePage = newCodePage;
-                                _defaultEncoding = System.Text.Encoding.GetEncoding(_defaultCodePage);
+                                _defaultEncoding = Encoding.UTF8;
+                            }
+                            else
+                            {
+                                int newCodePage = GetCodePage(env.newCollation, stateObj);
+                                if (newCodePage != _defaultCodePage)
+                                {
+                                    _defaultCodePage = newCodePage;
+                                    _defaultEncoding = System.Text.Encoding.GetEncoding(_defaultCodePage);
+                                }
                             }
                             _defaultLCID = env.newCollation.LCID;
                         }
@@ -3245,18 +3254,26 @@ internal bool TryProcessReturnValue(int length, TdsParserStateObject stateObj, o
                     return false;
                 }
 
-                int codePage = GetCodePage(rec.collation, stateObj);
-
-                // if the column lcid is the same as the default, use the default encoder
-                if (codePage == _defaultCodePage)
+                // UTF8 collation
+                if ((rec.collation.info & TdsEnums.UTF8_IN_TDSCOLLATION) == TdsEnums.UTF8_IN_TDSCOLLATION)
                 {
-                    rec.codePage = _defaultCodePage;
-                    rec.encoding = _defaultEncoding;
+                    rec.encoding = Encoding.UTF8;
                 }
                 else
                 {
-                    rec.codePage = codePage;
-                    rec.encoding = System.Text.Encoding.GetEncoding(rec.codePage);
+                    int codePage = GetCodePage(rec.collation, stateObj);
+
+                    // If the column lcid is the same as the default, use the default encoder
+                    if (codePage == _defaultCodePage)
+                    {
+                        rec.codePage = _defaultCodePage;
+                        rec.encoding = _defaultEncoding;
+                    }
+                    else
+                    {
+                        rec.codePage = codePage;
+                        rec.encoding = System.Text.Encoding.GetEncoding(rec.codePage);
+                    }
                 }
             }
 
@@ -3755,17 +3772,25 @@ private bool TryCommonProcessMetaData(TdsParserStateObject stateObj, _SqlMetaDat
                     return false;
                 }
 
-                int codePage = GetCodePage(col.collation, stateObj);
-
-                if (codePage == _defaultCodePage)
+                // UTF8 collation
+                if ((col.collation.info & TdsEnums.UTF8_IN_TDSCOLLATION) == TdsEnums.UTF8_IN_TDSCOLLATION)
                 {
-                    col.codePage = _defaultCodePage;
-                    col.encoding = _defaultEncoding;
+                    col.encoding = Encoding.UTF8;
                 }
                 else
                 {
-                    col.codePage = codePage;
-                    col.encoding = System.Text.Encoding.GetEncoding(col.codePage);
+                    int codePage = GetCodePage(col.collation, stateObj);
+
+                    if (codePage == _defaultCodePage)
+                    {
+                        col.codePage = _defaultCodePage;
+                        col.encoding = _defaultEncoding;
+                    }
+                    else
+                    {
+                        col.codePage = codePage;
+                        col.encoding = System.Text.Encoding.GetEncoding(col.codePage);
+                    }
                 }
             }
 
@@ -6067,6 +6092,19 @@ internal int WriteFedAuthFeatureRequest(FederatedAuthenticationFeatureExtensionD
 
             return len;
         }
+        internal int WriteUTF8SupportFeatureRequest(bool write /* if false just calculates the length */)
+        {
+            int len = 5; // 1byte = featureID, 4bytes = featureData length, sizeof(DWORD)
+
+            if (write)
+            {
+                // Write Feature ID
+                _physicalStateObj.WriteByte(TdsEnums.FEATUREEXT_UTF8SUPPORT);
+                WriteInt(0, _physicalStateObj); // we don't send any data
+            }
+
+            return len;
+        }
 
         internal void TdsLogin(SqlLogin rec, TdsEnums.FeatureExtension requestedFeatures, SessionData recoverySessionData, FederatedAuthenticationFeatureExtensionData? fedAuthFeatureExtensionData)
         {
@@ -6201,15 +6239,20 @@ internal void TdsLogin(SqlLogin rec, TdsEnums.FeatureExtension requestedFeatures
                 {
                     length += WriteSessionRecoveryFeatureRequest(recoverySessionData, false);
                 }
+                if ((requestedFeatures & TdsEnums.FeatureExtension.FedAuth) != 0)
+                {
+                    Debug.Assert(fedAuthFeatureExtensionData != null, "fedAuthFeatureExtensionData should not null.");
+                    length += WriteFedAuthFeatureRequest(fedAuthFeatureExtensionData.Value, write: false);
+                }
                 if ((requestedFeatures & TdsEnums.FeatureExtension.GlobalTransactions) != 0)
                 {
                     length += WriteGlobalTransactionsFeatureRequest(false);
                 }
-                if ((requestedFeatures & TdsEnums.FeatureExtension.FedAuth) != 0)
+                if ((requestedFeatures & TdsEnums.FeatureExtension.UTF8Support) != 0)
                 {
-                    Debug.Assert(fedAuthFeatureExtensionData != null, "fedAuthFeatureExtensionData should not null.");
-                    length += WriteFedAuthFeatureRequest(fedAuthFeatureExtensionData.Value, write: false);
+                    length += WriteUTF8SupportFeatureRequest(false);
                 }
+
                 length++; // for terminator
             }
 
@@ -6443,15 +6486,20 @@ internal void TdsLogin(SqlLogin rec, TdsEnums.FeatureExtension requestedFeatures
                     {
                         length += WriteSessionRecoveryFeatureRequest(recoverySessionData, true);
                     }
+                    if ((requestedFeatures & TdsEnums.FeatureExtension.FedAuth) != 0)
+                    {
+                        Debug.Assert(fedAuthFeatureExtensionData != null, "fedAuthFeatureExtensionData should not null.");
+                        WriteFedAuthFeatureRequest(fedAuthFeatureExtensionData.Value, write: true);
+                    }
                     if ((requestedFeatures & TdsEnums.FeatureExtension.GlobalTransactions) != 0)
                     {
                         WriteGlobalTransactionsFeatureRequest(true);
                     }
-                    if ((requestedFeatures & TdsEnums.FeatureExtension.FedAuth) != 0)
+                    if ((requestedFeatures & TdsEnums.FeatureExtension.UTF8Support) != 0)
                     {
-                        Debug.Assert(fedAuthFeatureExtensionData != null, "fedAuthFeatureExtensionData should not null.");
-                        WriteFedAuthFeatureRequest(fedAuthFeatureExtensionData.Value, write: true);
-                    };
+                        WriteUTF8SupportFeatureRequest(true);
+                    }
+
                     _physicalStateObj.WriteByte(0xFF); // terminator
                 }
             }
@@ -8127,6 +8175,12 @@ internal Task WriteBulkCopyValue(object value, SqlMetaDataPriv metadata, TdsPars
                 }
                 if (metadata.collation != null)
                 {
+                    // Replace encoding if it is UTF8
+                    if ((metadata.collation.info & TdsEnums.UTF8_IN_TDSCOLLATION) == TdsEnums.UTF8_IN_TDSCOLLATION)
+                    {
+                        _defaultEncoding = Encoding.UTF8;
+                    }
+
                     _defaultCollation = metadata.collation;
                     _defaultLCID = _defaultCollation.LCID;
                 }

src/System.Data.SqlClient/tests/ManualTests/SQL/Utf8SupportTest/Utf8SupportTest.cs

@@ -0,0 +1,35 @@
+using Xunit;
+
+namespace System.Data.SqlClient.ManualTesting.Tests
+{
+    public static class Utf8SupportTest
+    {
+        [CheckConnStrSetupFact]
+        public static void CheckSupportUtf8ConnectionProperty()
+        {
+            using (SqlConnection connection = new SqlConnection(DataTestUtility.TcpConnStr))
+            using (SqlCommand command = new SqlCommand())
+            {
+                command.Connection = connection;
+                command.CommandText = "SELECT CONNECTIONPROPERTY('SUPPORT_UTF8')";
+                connection.Open();
+
+                using (SqlDataReader reader = command.ExecuteReader())
+                {
+                    while (reader.Read())
+                    {
+                        // CONNECTIONPROPERTY('SUPPORT_UTF8') returns NULL in SQLServer versions that don't support UTF-8.
+                        if (!reader.IsDBNull(0))
+                        {
+                            Assert.Equal(1, reader.GetInt32(0));
+                        }
+                        else
+                        {
+                            Console.WriteLine("CONNECTIONPROPERTY('SUPPORT_UTF8') is not supported on this SQLServer");
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

src/System.Data.SqlClient/tests/ManualTests/System.Data.SqlClient.ManualTesting.Tests.csproj

@@ -121,6 +121,7 @@
     <Compile Include="SQL\UdtTest\UdtTest.cs" />
     <Compile Include="SQL\UdtTest\UdtTest2.cs" />
     <Compile Include="SQL\UdtTest\UdtTestHelpers.cs" />
+    <Compile Include="SQL\Utf8SupportTest\Utf8SupportTest.cs" />
     <Compile Include="SQL\WeakRefTest\WeakRefTest.cs" />
     <Compile Include="SQL\WeakRefTestYukonSpecific\WeakRefTestYukonSpecific.cs" />
     <Compile Include="XUnitAssemblyAttributes.cs" />