[release/dev17.13] Emit opted-in string literals into data section as UTF8 (#76785)

* Add feature flag for emitting string literals into data section as UTF8

* Address feedback

* Simplify some APIs

* Lift well known type member check

* Obtain the string type lazily

* Consolidate static constructor implementations

* Encapsulate IL emit of BytesToStringHelper

* Extract NestedTypeDefinition

* Share ParameterDefinitionBase

* Add more doc comments

* Remove unnecessary `using`s

* Avoid passing text and data together

* Avoid immutable array allocation

* Remove ITokenDeferral and SystemStringType passing

* Simplify parameter definition

* Cache threshold

* Simplify code

* Re-add Opt suffix

* Remove an unnecessary using

* Extend tests

* Improve tests

* Improve code

---------

Co-authored-by: Jan Jones <janjones@microsoft.com>

Author: github-actions[bot]

src/Compilers/CSharp/Portable/CSharpResources.resx

@@ -5309,7 +5309,7 @@ To remove the warning, you can use /reference instead (set the Embed Interop Typ
     <value>Syntax tree should be created from a submission.</value>
   </data>
   <data name="ERR_TooManyUserStrings" xml:space="preserve">
-    <value>Combined length of user strings used by the program exceeds allowed limit. Try to decrease use of string literals.</value>
+    <value>Combined length of user strings used by the program exceeds allowed limit. Try to decrease use of string literals or try the EXPERIMENTAL feature flag 'experimental-data-section-string-literals'.</value>
   </data>
   <data name="ERR_PatternNullableType" xml:space="preserve">
     <value>It is not legal to use nullable type '{0}?' in a pattern; use the underlying type '{0}' instead.</value>

src/Compilers/CSharp/Portable/CodeGen/EmitExpression.cs

@@ -3481,13 +3481,39 @@ private void EmitConstantExpression(TypeSymbol type, ConstantValue constantValue
                 {
                     EmitInitObj(type, used, syntaxNode);
                 }
-                else
+                else if (!TryEmitStringLiteralAsUtf8Encoded(constantValue, syntaxNode))
                 {
                     _builder.EmitConstantValue(constantValue);
                 }
             }
         }
 
+        private bool TryEmitStringLiteralAsUtf8Encoded(ConstantValue constantValue, SyntaxNode syntaxNode)
+        {
+            // Emit long strings into data section so they don't overflow the UserString heap.
+            if (constantValue.IsString &&
+                constantValue.StringValue.Length > _module.Compilation.DataSectionStringLiteralThreshold)
+            {
+                if (Binder.GetWellKnownTypeMember(_module.Compilation, WellKnownMember.System_Text_Encoding__get_UTF8, _diagnostics, syntax: syntaxNode) == null |
+                    Binder.GetWellKnownTypeMember(_module.Compilation, WellKnownMember.System_Text_Encoding__GetString, _diagnostics, syntax: syntaxNode) == null)
+                {
+                    return false;
+                }
+
+                Cci.IFieldReference field = _module.TryGetOrCreateFieldForStringValue(constantValue.StringValue, syntaxNode, _diagnostics.DiagnosticBag);
+                if (field == null)
+                {
+                    return false;
+                }
+
+                _builder.EmitOpCode(ILOpCode.Ldsfld);
+                _builder.EmitToken(field, syntaxNode, _diagnostics.DiagnosticBag);
+                return true;
+            }
+
+            return false;
+        }
+
         private void EmitInitObj(TypeSymbol type, bool used, SyntaxNode syntaxNode)
         {
             if (used)

src/Compilers/CSharp/Portable/Compiler/MethodCompiler.cs

@@ -671,8 +671,11 @@ private void CompileSynthesizedMethods(PrivateImplementationDetails privateImplC
             foreach (Cci.IMethodDefinition definition in privateImplClass.GetMethods(context).Concat(privateImplClass.GetTopLevelAndNestedTypeMethods(context)))
             {
                 var method = (MethodSymbol)definition.GetInternalSymbol();
-                Debug.Assert(method.SynthesizesLoweredBoundBody);
-                method.GenerateMethodBody(compilationState, diagnostics);
+                if (method is not null)
+                {
+                    Debug.Assert(method.SynthesizesLoweredBoundBody);
+                    method.GenerateMethodBody(compilationState, diagnostics);
+                }
             }
 
             CompileSynthesizedMethods(compilationState);

src/Compilers/CSharp/Portable/Lowering/LocalRewriter/LocalRewriter_Conversion.cs

@@ -122,18 +122,16 @@ private BoundExpression MakeUtf8Span(BoundExpression node, IReadOnlyList<byte>?
 
         private byte[]? GetUtf8ByteRepresentation(BoundUtf8String node)
         {
-            var utf8 = new System.Text.UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
-
-            try
+            if (node.Value.TryGetUtf8ByteRepresentation(out byte[]? result, out string? error))
             {
-                return utf8.GetBytes(node.Value);
+                return result;
             }
-            catch (Exception ex)
+            else
             {
                 _diagnostics.Add(
                     ErrorCode.ERR_CannotBeConvertedToUtf8,
                     node.Syntax.Location,
-                    ex.Message);
+                    error);
 
                 return null;
             }