Additional parameter validation for StartNoGCRegion (#14088)

* Additional parameter validation for StartNoGCRegion

* Fix the clang build by moving some defs before the first goto

* Guard against size_t overflow when scaling allocation budgets

* Move some parameter validation for managed code, throw better exceptions on parameter validation errors

* Remove stray comma

* Address code review feedback - unify code branches that branch to the failure case. Also keep data types as uint64_t as long as possible to avoid size_t overflow

* Fix the clang build

* Code review feedback:
 1. Remove useless asserts
 2. Remove a dead code path
 3. Add a LocalGC TODO to comment

Author: swgillespie

src/gc/gc.cpp

@@ -15880,45 +15880,71 @@ start_no_gc_region_status gc_heap::prepare_for_no_gc_region (uint64_t total_size
     save_data_for_no_gc();
     settings.pause_mode = pause_no_gc;
     current_no_gc_region_info.start_status = start_no_gc_success;
-    
-    size_t allocation_no_gc_loh = 0;
-    size_t allocation_no_gc_soh = 0;
-    size_t size_per_heap = 0;
 
-    total_size = (size_t)((float)total_size * 1.05);
+    uint64_t allocation_no_gc_loh = 0;
+    uint64_t allocation_no_gc_soh = 0;
+    assert(total_size != 0);
     if (loh_size_known)
     {
-        loh_size = (size_t)((float)loh_size * 1.05);
-        allocation_no_gc_loh = (size_t)loh_size;
-        allocation_no_gc_soh = (size_t)(total_size - loh_size);
+        assert(loh_size != 0);
+        assert(loh_size <= total_size);
+        allocation_no_gc_loh = loh_size;
+        allocation_no_gc_soh = total_size - loh_size;
     }
     else
     {
-        allocation_no_gc_soh = (size_t)total_size;
-        allocation_no_gc_loh = (size_t)total_size;
+        allocation_no_gc_soh = total_size;
+        allocation_no_gc_loh = total_size;
     }
 
     int soh_align_const = get_alignment_constant (TRUE);
-    size_t max_soh_allocated = (soh_segment_size - segment_info_size - eph_gen_starts_size);
+    size_t max_soh_allocated = soh_segment_size - segment_info_size - eph_gen_starts_size;
+    size_t size_per_heap = 0;
+    const double scale_factor = 1.05;
 
     int num_heaps = 1;
 #ifdef MULTIPLE_HEAPS
     num_heaps = n_heaps;
-#endif //MULTIPLE_HEAPS
-    size_t total_allowed_soh_allocation = max_soh_allocated * num_heaps;
+#endif // MULTIPLE_HEAPS
 
-    if (allocation_no_gc_soh > total_allowed_soh_allocation)
+    uint64_t total_allowed_soh_allocation = max_soh_allocated * num_heaps;
+    // [LOCALGC TODO]
+    // In theory, the upper limit here is the physical memory of the machine, not
+    // SIZE_T_MAX. This is not true today because total_physical_mem can be
+    // larger than SIZE_T_MAX if running in wow64 on a machine with more than
+    // 4GB of RAM. Once Local GC code divergence is resolved and code is flowing
+    // more freely between branches, it would be good to clean this up to use
+    // total_physical_mem instead of SIZE_T_MAX.
+    assert(total_allowed_soh_allocation <= SIZE_T_MAX);
+    uint64_t total_allowed_loh_allocation = SIZE_T_MAX;
+    uint64_t total_allowed_soh_alloc_scaled = allocation_no_gc_soh > 0 ? static_cast<uint64_t>(total_allowed_soh_allocation / scale_factor) : 0;
+    uint64_t total_allowed_loh_alloc_scaled = allocation_no_gc_loh > 0 ? static_cast<uint64_t>(total_allowed_loh_allocation / scale_factor) : 0;
+
+    if (allocation_no_gc_soh > total_allowed_soh_alloc_scaled ||
+        allocation_no_gc_loh > total_allowed_loh_alloc_scaled)
     {
         status = start_no_gc_too_large;
         goto done;
     }
 
+    if (allocation_no_gc_soh > 0)
+    {
+        allocation_no_gc_soh = static_cast<uint64_t>(allocation_no_gc_soh * scale_factor);
+        allocation_no_gc_soh = min (allocation_no_gc_soh, total_allowed_soh_alloc_scaled);
+    }
+
+    if (allocation_no_gc_loh > 0)
+    {
+        allocation_no_gc_loh = static_cast<uint64_t>(allocation_no_gc_loh * scale_factor);
+        allocation_no_gc_loh = min (allocation_no_gc_loh, total_allowed_loh_alloc_scaled);
+    }
+
     if (disallow_full_blocking)
         current_no_gc_region_info.minimal_gc_p = TRUE;
 
     if (allocation_no_gc_soh != 0)
     {
-        current_no_gc_region_info.soh_allocation_size = allocation_no_gc_soh;
+        current_no_gc_region_info.soh_allocation_size = static_cast<size_t>(allocation_no_gc_soh);
         size_per_heap = current_no_gc_region_info.soh_allocation_size;
 #ifdef MULTIPLE_HEAPS
         size_per_heap /= n_heaps;
@@ -15934,7 +15960,7 @@ start_no_gc_region_status gc_heap::prepare_for_no_gc_region (uint64_t total_size
 
     if (allocation_no_gc_loh != 0)
     {
-        current_no_gc_region_info.loh_allocation_size = allocation_no_gc_loh;
+        current_no_gc_region_info.loh_allocation_size = static_cast<size_t>(allocation_no_gc_loh);
         size_per_heap = current_no_gc_region_info.loh_allocation_size;
 #ifdef MULTIPLE_HEAPS
         size_per_heap /= n_heaps;

src/mscorlib/src/System/GC.cs

@@ -24,6 +24,7 @@
 using System.Globalization;
 using System.Runtime.InteropServices;
 using System.Runtime.Versioning;
+using System.Diagnostics;
 
 namespace System
 {
@@ -434,14 +435,37 @@ private enum EndNoGCRegionStatus
 
         private static bool StartNoGCRegionWorker(long totalSize, bool hasLohSize, long lohSize, bool disallowFullBlockingGC)
         {
+            if (totalSize <= 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(totalSize), "totalSize can't be zero or negative");
+            }
+
+            if (hasLohSize)
+            {
+                if (lohSize <= 0)
+                {
+                    throw new ArgumentOutOfRangeException(nameof(lohSize), "lohSize can't be zero or negative");
+                }
+
+                if (lohSize > totalSize)
+                {
+                    throw new ArgumentOutOfRangeException(nameof(lohSize), "lohSize can't be greater than totalSize");
+                }
+            }
+
             StartNoGCRegionStatus status = (StartNoGCRegionStatus)_StartNoGCRegion(totalSize, hasLohSize, lohSize, disallowFullBlockingGC);
-            if (status == StartNoGCRegionStatus.AmountTooLarge)
-                throw new ArgumentOutOfRangeException(nameof(totalSize),
-                    "totalSize is too large. For more information about setting the maximum size, see \"Latency Modes\" in http://go.microsoft.com/fwlink/?LinkId=522706");
-            else if (status == StartNoGCRegionStatus.AlreadyInProgress)
-                throw new InvalidOperationException("The NoGCRegion mode was already in progress");
-            else if (status == StartNoGCRegionStatus.NotEnoughMemory)
-                return false;
+            switch (status)
+            {
+                case StartNoGCRegionStatus.NotEnoughMemory:
+                    return false;
+                case StartNoGCRegionStatus.AlreadyInProgress:
+                    throw new InvalidOperationException("The NoGCRegion mode was already in progress");
+                case StartNoGCRegionStatus.AmountTooLarge:
+                    throw new ArgumentOutOfRangeException(nameof(totalSize),
+                        "totalSize is too large. For more information about setting the maximum size, see \"Latency Modes\" in http://go.microsoft.com/fwlink/?LinkId=522706");
+            }
+
+            Debug.Assert(status == StartNoGCRegionStatus.Succeeded);
             return true;
         }