Error out when struct size is bigger than int.MaxValue

src/coreclr/tools/Common/TypeSystem/Common/LayoutInt.cs

@@ -92,6 +92,20 @@ public string ToStringInvariant()
             return new LayoutInt(checked(left._value - right._value));
         }
 
+        public static LayoutInt AddThrowing(LayoutInt left, LayoutInt right, TypeDesc loadedType)
+        {
+            if (left.IsIndeterminate || right.IsIndeterminate)
+                return Indeterminate;
+
+            int result = left._value + right._value;
+
+            // Overflow if both arguments have the opposite sign of the result
+            if (((left._value ^ result) & (right._value ^ result)) < 0)
+                ThrowHelper.ThrowTypeLoadException(loadedType);
+
+            return new LayoutInt(result);
+        }
+
         public override bool Equals(object obj)
         {
             if (obj is LayoutInt)

src/coreclr/tools/Common/TypeSystem/Common/MetadataFieldLayoutAlgorithm.cs

@@ -436,7 +436,7 @@ protected ComputedInstanceFieldLayout ComputeSequentialFieldLayout(MetadataType
 
                 cumulativeInstanceFieldPos = AlignUpInstanceFieldOffset(cumulativeInstanceFieldPos, fieldSizeAndAlignment.Alignment, type.Context.Target);
                 offsets[fieldOrdinal] = new FieldAndOffset(field, cumulativeInstanceFieldPos + offsetBias);
-                cumulativeInstanceFieldPos = checked(cumulativeInstanceFieldPos + fieldSizeAndAlignment.Size);
+                cumulativeInstanceFieldPos = LayoutInt.AddThrowing(cumulativeInstanceFieldPos, fieldSizeAndAlignment.Size, type);
 
                 fieldOrdinal++;
             }

src/coreclr/tools/Common/TypeSystem/IL/Stubs/ComparerIntrinsics.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Collections;
 using System.Collections.Generic;
 
 using Internal.TypeSystem;
@@ -301,11 +302,11 @@ public static bool CanCompareValueTypeBits(MetadataType type, MethodDesc objectE
 
         private struct OverlappingFieldTracker
         {
-            private bool[] _usedBytes;
+            private BitArray _usedBytes;
 
             public OverlappingFieldTracker(MetadataType type)
             {
-                _usedBytes = new bool[type.InstanceFieldSize.AsInt];
+                _usedBytes = new BitArray(type.InstanceFieldSize.AsInt);
             }
 
             public bool TrackField(FieldDesc field)

src/coreclr/vm/methodtablebuilder.cpp

@@ -1764,6 +1764,9 @@ MethodTableBuilder::BuildMethodTableThrowing(
 
         bmtFP->NumInstanceFieldBytes = GetLayoutInfo()->m_cbManagedSize;
 
+        if ((int)bmtFP->NumInstanceFieldBytes != (INT64)bmtFP->NumInstanceFieldBytes)
+            BuildMethodTableThrowException(IDS_CLASSLOAD_FIELDTOOLARGE);
+
         // For simple Blittable types we still need to check if they have any overlapping
         // fields and call the method SetHasOverlaidFields() when they are detected.
         //

src/tests/Loader/classloader/SequentialLayout/ManagedSequential/LargeStructSize.cs

@@ -0,0 +1,36 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using Xunit;
+
+[SkipOnMono("This test suite tests CoreCLR and Crossgen2/NativeAOT-specific layout rules.")]
+public unsafe class LargeStructSize
+{
+    struct X
+    {
+        byte x;
+        BigArray a;
+    }
+
+    struct Y
+    {
+        BigArray a;
+        byte y;
+    }
+
+    [StructLayout(LayoutKind.Sequential, Size = int.MaxValue)]
+    struct BigArray
+    {
+    }
+
+    [Fact]
+    public static void TestLargeStructSize()
+    {
+        Assert.Equal(int.MaxValue, sizeof(BigArray));
+        Assert.Throws<TypeLoadException>(() => sizeof(X));
+        Assert.Throws<TypeLoadException>(() => sizeof(Y));
+    }
+}

src/tests/Loader/classloader/SequentialLayout/ManagedSequential/LargeStructSize.csproj

@@ -0,0 +1,8 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+  </ItemGroup>
+</Project>