RFC: Avoid running out of memory by checking available memory before allocating

Make.inc

@@ -1151,7 +1151,7 @@ LIBGFORTRAN_VERSION := $(subst libgfortran,,$(filter libgfortran%,$(subst -,$(SP
 # Note: we explicitly _do not_ define `CSL` here, since it requires some more
 # advanced techniques to decide whether it should be installed from a BB source
 # or not.  See `deps/csl.mk` for more detail.
-BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP
+BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP
 define SET_BB_DEFAULT
 # First, check to see if BB is disabled on a global setting
 ifeq ($$(USE_BINARYBUILDER),0)

base/sysinfo.jl

@@ -20,6 +20,8 @@ export BINDIR,
        loadavg,
        free_memory,
        total_memory,
+       physical_free_memory,
+       physical_total_memory,
        isapple,
        isbsd,
        isdragonfly,
@@ -246,19 +248,44 @@ function loadavg()
     return loadavg_
 end
 
+"""
+    Sys.free_physical_memory()
+
+Get the total memory in RAM (including that which is currently used) in bytes. The entire
+amount may not be available to the current process; see `Sys.total_memory()`.
+"""
+free_physical_memory() = ccall(:uv_get_free_memory, UInt64, ())
+
+"""
+    Sys.total_physical_memory()
+
+Get the total memory in RAM (including that which is currently used) in bytes. The entire
+amount may not be available to the current process; see `Sys.total_memory()`.
+"""
+total_physical_memory() = ccall(:uv_get_total_memory, UInt64, ())
+
 """
     Sys.free_memory()
 
 Get the total free memory in RAM in bytes.
 """
-free_memory() = ccall(:uv_get_free_memory, UInt64, ())
+free_memory() = ccall(:uv_get_available_memory, UInt64, ())
 
 """
     Sys.total_memory()
 
 Get the total memory in RAM (including that which is currently used) in bytes.
+This amount may be constrained, e.g., by Linux control groups. For the unconstrained
+amount, see `Sys.physical_memory()`.
 """
-total_memory() = ccall(:uv_get_total_memory, UInt64, ())
+function total_memory()
+    memory = ccall(:uv_get_constrained_memory, UInt64, ())
+    if memory == 0
+        return total_physical_memory()
+    else
+        return memory
+    end
+end
 
 """
     Sys.get_process_title()

deps/checksums/libuv

@@ -1,34 +1,34 @@
-LibUV.v2.0.1+8.aarch64-apple-darwin.tar.gz/md5/c6123b5807b457a7b171b9060bcafa19
-LibUV.v2.0.1+8.aarch64-apple-darwin.tar.gz/sha512/bfbf31fde87e8a4bbb9cde72fba98c562a66a5c64cb2a9998dce4f94cc955fd6afa0b54757682499da483baee9c78c30bd685a60ef6419d2b7383fd313f7eee3
-LibUV.v2.0.1+8.aarch64-linux-gnu.tar.gz/md5/97274d22abb4c3674508732907d74b47
-LibUV.v2.0.1+8.aarch64-linux-gnu.tar.gz/sha512/2adbfaaf690d928b7d32b2e4d48a53c4ffdd94cb699db8e46d93dde496a33b29feb3f0d1c62df42b2dfaace9941a79829d54fd68900632f9cec5da71015de28f
-LibUV.v2.0.1+8.aarch64-linux-musl.tar.gz/md5/0eaec69cc9b40d99c23182b7a20c4b81
-LibUV.v2.0.1+8.aarch64-linux-musl.tar.gz/sha512/224156e8fb287d45060445dbbc2dedafebee0cd44923b541c13d6688c8e8f7a86fe608fe6235c9459a2f07eac9e4b0d38577164674238f89033c3ab8c76e6e05
-LibUV.v2.0.1+8.armv6l-linux-gnueabihf.tar.gz/md5/2ddd26fac1ec25faa44be79a95ea52e0
-LibUV.v2.0.1+8.armv6l-linux-gnueabihf.tar.gz/sha512/123a1faf182e4e757b96faf2f4981f4985246780796e0be9239872dbcc76631f2d028171a6e40150b532b4840de83c36e274e9630c2474ed50e9c150efaf2dd7
-LibUV.v2.0.1+8.armv6l-linux-musleabihf.tar.gz/md5/bf474e3faa0a8dafdc3c37eef1f22133
-LibUV.v2.0.1+8.armv6l-linux-musleabihf.tar.gz/sha512/9a4e4b0af14e5e16e654033f2b77910a5004dbbd52eaad844429af7f521233a8a352d3f12e96a5d6dc6b709b578146d9bb34e12ba959b0cc111b8a6667fc88d9
-LibUV.v2.0.1+8.armv7l-linux-gnueabihf.tar.gz/md5/a10c8d87b4cc631e85d93c3e0ea0e882
-LibUV.v2.0.1+8.armv7l-linux-gnueabihf.tar.gz/sha512/65ebe30c7e14a4d72e0489cfcc299a45a6313102b540b2c245df0d098ec9c79c1037517c9a341b095912fe81c36fd6d5c308dfb090169dea76c04105c871e790
-LibUV.v2.0.1+8.armv7l-linux-musleabihf.tar.gz/md5/d77772d6330ae6692fd1295f3dfea8a8
-LibUV.v2.0.1+8.armv7l-linux-musleabihf.tar.gz/sha512/c21ab143bb5262fb09a8d457ef53f639e3040802abd128bb49b300015bba857fe3adaa0181e277b7a79ca20e02aaafc644f1297cfc18a1e1ca8af8cf1af711be
-LibUV.v2.0.1+8.i686-linux-gnu.tar.gz/md5/54bb6813c26a7e0ea2518de5faa243a5
-LibUV.v2.0.1+8.i686-linux-gnu.tar.gz/sha512/cef37e6b164a66135bb5eb3742827575a41e0723fa66e037b030833461bec681054c70684d0ab3b30e52a5b0a16eb003cc9a67003652f0865b7e0d504af2e7a8
-LibUV.v2.0.1+8.i686-linux-musl.tar.gz/md5/515fbd2e524ae8bff39520fa50ebe792
-LibUV.v2.0.1+8.i686-linux-musl.tar.gz/sha512/5b5679937c4aef39fc22bb8681440a33bd53eb6115315f8b169f65a9f59d632f3d774b0cd3fe14d9a2f74db64a459f0e81ceb94648c6c464e0d275567c87dadb
-LibUV.v2.0.1+8.i686-w64-mingw32.tar.gz/md5/7f0fedba47d432c48b26757348b1eb5d
-LibUV.v2.0.1+8.i686-w64-mingw32.tar.gz/sha512/3d3fe9bbd210896d68d3de2748a013a59e11fa42360c186fe7a461e2aa4b8c26fa7bacd1a04cd22e86c8600b2d764cb5c66a0cacbf956df8df04aa6cf26503f7
-LibUV.v2.0.1+8.powerpc64le-linux-gnu.tar.gz/md5/feac1f65834f86b0f1aedf002431fbd4
-LibUV.v2.0.1+8.powerpc64le-linux-gnu.tar.gz/sha512/f78b55da9ee0c9cd13c4824e07d4b96f7b47dfbc2d7abc5d09210cfff3c659f46ebb3dc733f8eeb9c6464c8252927e89f76540b48cdb370dd89e8ea1eabc6cb8
-LibUV.v2.0.1+8.x86_64-apple-darwin.tar.gz/md5/930a03b3cb44a2a42ff20be4c5bb388d
-LibUV.v2.0.1+8.x86_64-apple-darwin.tar.gz/sha512/5398264f42707c35cacb68ba5dab84e49efcb5571e01b16055030dd11f5b8ea4371972b419e2087a1daf9565b4b578633fce13d1e050adb23f91b7ac16ad1937
-LibUV.v2.0.1+8.x86_64-linux-gnu.tar.gz/md5/3a3346f4e91123d49bf41a124303a670
-LibUV.v2.0.1+8.x86_64-linux-gnu.tar.gz/sha512/d81951bb396e5116d80127a69546844ec99f9b19b229a0344d3f9051e2f08c13f3abb0650bc314ca5be0fc556f0a655dba960e2513de590db0bfaae4575d6f54
-LibUV.v2.0.1+8.x86_64-linux-musl.tar.gz/md5/603fec3ba7efb51be040c22803784ded
-LibUV.v2.0.1+8.x86_64-linux-musl.tar.gz/sha512/345fe3f0cedf7404345d49e9eca7032787d758b3d5e2e0af2b59836a1fc2e5b71738c8389eb31738c161527a717c7f6f30af123e3a9056785bcdbcb9a1b18057
-LibUV.v2.0.1+8.x86_64-unknown-freebsd.tar.gz/md5/ec6df758c4b27a495adb97820b5a8b22
-LibUV.v2.0.1+8.x86_64-unknown-freebsd.tar.gz/sha512/16d3c39d8fd2e4c9035cff99433c5f5a9150da18fac3ed9639e22a38715ef51f30d28f83674c68be69d1305f4b53b785f3a8f89d28253fe254bd2515fc49d5ea
-LibUV.v2.0.1+8.x86_64-w64-mingw32.tar.gz/md5/0dff06ff0a42ac3404f78f7eccf08f1b
-LibUV.v2.0.1+8.x86_64-w64-mingw32.tar.gz/sha512/4884c6cd7c29eee725b0d9f6eab091086665b3070f3cd33854f482c9b2ec55924ab560bd5d27b46c5e0ed6a7af08bffb5beb0e75fef6aa82b4f90bd7599ee691
-libuv-3f7038d62e43c3682394a6ea7b4ccc46be0fa0bf.tar.gz/md5/fe6957e2603df688a40605134505052b
-libuv-3f7038d62e43c3682394a6ea7b4ccc46be0fa0bf.tar.gz/sha512/28f13b8d927e0663bff26e9ab829538947754046088a42fc85d772d62c28fb47f8f3075d1a884bbbae6ce0ba294eb8f3b5f9cb95be14d7ae1f467b713b4a14a7
+LibUV.v2.0.1+10.aarch64-apple-darwin.tar.gz/md5/a93812c22486fbc7cfab186a8bb943e9
+LibUV.v2.0.1+10.aarch64-apple-darwin.tar.gz/sha512/06d612c7cd8d6996c8181f742120b05327dcd83de55be8a8fd2098f67093a26c8aac7de96e230c7462b45ac45c411c9e62680e8ffdcb5d5814eba68e95fc0c96
+LibUV.v2.0.1+10.aarch64-linux-gnu.tar.gz/md5/f7121653c0491970e78556a7ecab640f
+LibUV.v2.0.1+10.aarch64-linux-gnu.tar.gz/sha512/a5309fe5052e166ab2aed10c69526785b8680b61d21c0522e9abe936a728cc1ff67e7b2bd0c2a6367a3af57542b26ea04568a6a88e3596722e9487311789fc0f
+LibUV.v2.0.1+10.aarch64-linux-musl.tar.gz/md5/372e4d2eb0ec94aa68e1bfabff6bb9c4
+LibUV.v2.0.1+10.aarch64-linux-musl.tar.gz/sha512/7655f532287d228c11fdbc47331f562a7114a290736fd028650098e17366edb5c34767a8c2e273bb44555a283dabde135d1276c42a33b8936f5169dd6f8c24bd
+LibUV.v2.0.1+10.armv6l-linux-gnueabihf.tar.gz/md5/06cdd23fb9a77d1294c5af9f087b7087
+LibUV.v2.0.1+10.armv6l-linux-gnueabihf.tar.gz/sha512/279ef1e500d77663681d0372d3a91d8a6eac7f8fb023aa3e2023259b0c18d357e9ed1acd9d6d3764e51a1a498b1dd8e3079f71fd874cb3090700c87afe4ec1e5
+LibUV.v2.0.1+10.armv6l-linux-musleabihf.tar.gz/md5/5b920ea5dbdec39aafc3e1027c317ca1
+LibUV.v2.0.1+10.armv6l-linux-musleabihf.tar.gz/sha512/a39a7617659aa64c3b8c8ccce38164c9d010cbf642bfb61eb26653a6f2fe32bca2ad6da8fe23c63e4149298563a23c76b184e91a06935a49201eae24f4026a42
+LibUV.v2.0.1+10.armv7l-linux-gnueabihf.tar.gz/md5/5db8612d4242130e54706707f2329c89
+LibUV.v2.0.1+10.armv7l-linux-gnueabihf.tar.gz/sha512/d85114037ef559ab5e291cfb5be5eea0bc1939e6e8a696c9861f32176f680d84d92cb717bcefaa2cc1ba96d8d16322e821edd421cedb2c828f3df7d5478f28e4
+LibUV.v2.0.1+10.armv7l-linux-musleabihf.tar.gz/md5/510d26564fbfc646a099a6709ae9dbf6
+LibUV.v2.0.1+10.armv7l-linux-musleabihf.tar.gz/sha512/91073781073059d15d1ddceeb4401c944dc6f4d56f2a4122b81d39f2cc5b497d4dc2b7b1da8e49ee612e5de41dc8d6e37b1f6324e4d14aef0580ee6dfa2a0454
+LibUV.v2.0.1+10.i686-linux-gnu.tar.gz/md5/6d1d33d5513a63aa75cd1bf5ae323343
+LibUV.v2.0.1+10.i686-linux-gnu.tar.gz/sha512/59fc3a683a74b04aecb7ef8bd43aba956f0f7c4908bae39071c8dfcd9fd6f811bbf31aeadf6c698e0fa3a4289b216fe8742110a821725eaeb34f92b38566a8c4
+LibUV.v2.0.1+10.i686-linux-musl.tar.gz/md5/712fcf6d2ea762706f7b35a34c646f40
+LibUV.v2.0.1+10.i686-linux-musl.tar.gz/sha512/59de39f00e3149af5dd43da54c77dc90cc8a9906613c50a3e15d8f6e614a93b6baab6f47fbc4e4a317fa3d576e00ec07c234bcf89d7eed36848d6002ffd657c9
+LibUV.v2.0.1+10.i686-w64-mingw32.tar.gz/md5/75cbd8a472dde1b85552b0e7f6a65e5c
+LibUV.v2.0.1+10.i686-w64-mingw32.tar.gz/sha512/1dd1203bc9daeed4b2d8e6b8a45b91cfb2b788519645b48806bd141256de71301de2c1344a4e9e744ec000a31258038623924b77dec5575a412ad1fe4935979a
+LibUV.v2.0.1+10.powerpc64le-linux-gnu.tar.gz/md5/ab06f8f8aaae51f4fdff225bd903fe06
+LibUV.v2.0.1+10.powerpc64le-linux-gnu.tar.gz/sha512/18c466e561a1e542c30631c5dd0f25fc546879d09727b5a469867288fc781f0fb785aa5de009541bda4397eb84e829e3252a35d4ad0a1c79a38939e8cfe2d05b
+LibUV.v2.0.1+10.x86_64-apple-darwin.tar.gz/md5/56c99b1611032cf1bdb77a64ca6ce6f7
+LibUV.v2.0.1+10.x86_64-apple-darwin.tar.gz/sha512/e297cd46f8286d2056bf148281fb5ee684eef08446c0136db9df684e5a20a241e1294249a6547664314f8f631e286184ee470986efc8bdb13fd08a798cfcd369
+LibUV.v2.0.1+10.x86_64-linux-gnu.tar.gz/md5/210365e89ac264e2ed8c043957860f98
+LibUV.v2.0.1+10.x86_64-linux-gnu.tar.gz/sha512/4583a0ee1bc3dee8f6abecd18516d493b88a791922e8e4a78d268e72575cfde77b54f25bf9b10e9b06a9b6a19ebf54489c2560a7048bf1d469a58eda5362eaea
+LibUV.v2.0.1+10.x86_64-linux-musl.tar.gz/md5/4d1d668abd7585f7bdc344a193fc56bb
+LibUV.v2.0.1+10.x86_64-linux-musl.tar.gz/sha512/219c6c6f5adff897b8e72419a69249e5199dcda36d8ef7a915805abe553c1e44648f723f569f1cbaf636f86efe2df293626f74931970a2eb42f506f26f69af1f
+LibUV.v2.0.1+10.x86_64-unknown-freebsd.tar.gz/md5/50de1c6e53fc09525f878434e4efa978
+LibUV.v2.0.1+10.x86_64-unknown-freebsd.tar.gz/sha512/184e006550a9464062bb09f955f2c969351f916a18ab9bed5c8643cde877a5bfcbae6acf65b303b915717b7c4a0e32607f17aa1deb256ddc4bd3ae5b02933f26
+LibUV.v2.0.1+10.x86_64-w64-mingw32.tar.gz/md5/054685997cf3ba454d947aa7030655e5
+LibUV.v2.0.1+10.x86_64-w64-mingw32.tar.gz/sha512/d01d535a2f51f20d2203add814293113607c3904d657d7c57c5b3f60667dc629ebc1ee8ef04cb32e8f702675cd302af18f41bf7992142d4a2a9aacc4a4ef12e2
+libuv-6d2f161c9b23d9f188fc67ecf04b23a2bd146a7c.tar.gz/md5/384beea8829d292f58446ddbb5e10e0e
+libuv-6d2f161c9b23d9f188fc67ecf04b23a2bd146a7c.tar.gz/sha512/248019e82f83ce6948d7226c5d334d1b68dfb12e4e3fa9249a03c9711b6adae2c9bc6f8a9124be0afe40b4d6082cc6081ba681c29e5dcba697154ca3661168c2

deps/checksums/libuv-e5928992b60d4642742d61d6257de09eaeb2e572.tar.gz/md5

@@ -0,0 +1 @@
+638fd38874c9b7224c0d3e0a3d0eee4f

deps/checksums/libuv-e5928992b60d4642742d61d6257de09eaeb2e572.tar.gz/sha512

@@ -0,0 +1 @@
+4ec1a4fa81c04bb159e5c7299460900f3921d644224408804688cb767400003e1335067767ae7e73a134c2c753707785017114bce305ecc414edf62d1d514aa3

deps/libuv.version

@@ -3,5 +3,5 @@ LIBUV_JLL_NAME := LibUV
 
 ## source build
 LIBUV_VER := 2
-LIBUV_BRANCH=julia-uv2-1.44.2
-LIBUV_SHA1=3f7038d62e43c3682394a6ea7b4ccc46be0fa0bf
+LIBUV_BRANCH=tb/available_memory
+LIBUV_SHA1=e5928992b60d4642742d61d6257de09eaeb2e572

src/gc.c

@@ -227,6 +227,8 @@ STATIC_INLINE void jl_free_aligned(void *p) JL_NOTSAFEPOINT
 #else
 STATIC_INLINE void *jl_malloc_aligned(size_t sz, size_t align)
 {
+    if (uv_get_available_memory() < sz)
+        jl_throw(jl_memory_exception);
 #if defined(_P64) || defined(__APPLE__)
     if (align <= 16)
         return malloc(sz);
@@ -3486,15 +3488,15 @@ void jl_gc_init(void)
     // on a big memory machine, set max_collect_interval to totalmem / nthreads / 2
     uint64_t total_mem = uv_get_total_memory();
     uint64_t constrained_mem = uv_get_constrained_memory();
-    if (constrained_mem > 0 && constrained_mem < total_mem)
+    if (constrained_mem != 0)
         total_mem = constrained_mem;
     size_t maxmem = total_mem / jl_n_threads / 2;
     if (maxmem > max_collect_interval)
         max_collect_interval = maxmem;
 #endif
 
     // We allocate with abandon until we get close to the free memory on the machine.
-    uint64_t free_mem = uv_get_free_memory();
+    uint64_t free_mem = uv_get_available_memory();
     uint64_t high_water_mark = free_mem / 10 * 7;  // 70% high water mark
 
     if (high_water_mark < max_total_memory)
@@ -3532,6 +3534,8 @@ JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz)
         jl_atomic_store_relaxed(&ptls->gc_num.malloc,
             jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1);
     }
+    if (uv_get_available_memory() < sz)
+        jl_throw(jl_memory_exception);
     return malloc(sz);
 }
 
@@ -3547,6 +3551,8 @@ JL_DLLEXPORT void *jl_gc_counted_calloc(size_t nm, size_t sz)
         jl_atomic_store_relaxed(&ptls->gc_num.malloc,
             jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1);
     }
+    if (uv_get_available_memory() < sz)
+        jl_throw(jl_memory_exception);
     return calloc(nm, sz);
 }
 
@@ -3696,6 +3702,8 @@ static void *gc_managed_realloc_(jl_ptls_t ptls, void *d, size_t sz, size_t olds
 #ifdef _OS_WINDOWS_
     DWORD last_error = GetLastError();
 #endif
+    if (allocsz > oldsz && uv_get_available_memory() < (allocsz-oldsz))
+        jl_throw(jl_memory_exception);
     void *b;
     if (isaligned)
         b = realloc_cache_align(d, allocsz, oldsz);
@@ -3775,6 +3783,8 @@ static void *gc_perm_alloc_large(size_t sz, int zero, unsigned align, unsigned o
 #ifdef _OS_WINDOWS_
     DWORD last_error = GetLastError();
 #endif
+    if (uv_get_available_memory() < sz)
+        jl_throw(jl_memory_exception);
     uintptr_t base = (uintptr_t)(zero ? calloc(1, sz) : malloc(sz));
     if (base == 0)
         jl_throw(jl_memory_exception);

stdlib/LibUV_jll/Project.toml

@@ -1,6 +1,6 @@
 name = "LibUV_jll"
 uuid = "183b4373-6708-53ba-ad28-60e28bb38547"
-version = "2.0.1+8"
+version = "2.0.1+10"
 
 [deps]
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"