Auto merge of #31846 - alexcrichton:better-disable-jemallc, r=brson

The `--disable-jemalloc` configure option has a failure mode where it will
create a distribution that is not compatible with other compilers. For example
the nightly for Linux will assume that it will link to jemalloc by default as
an allocator for executable crates. If, however, a standard library is used
which was built via `./configure --disable-jemalloc` then this will fail
because the jemalloc crate wasn't built.

While this seems somewhat reasonable as a niche situation, the same mechanism is
used for disabling jemalloc for platforms that just don't support it. For
example if the rumprun target is compiled then the sibiling Linux target *also*
doesn't have jemalloc. This is currently a problem for our cross-build nightlies
which build many targets. If rumprun is also built, it will disable jemalloc for
all targets, which isn't desired.

This commit moves the platform-specific disabling of jemalloc as hardcoded logic
into the makefiles that is scoped per-platform. This way when configuring
multiple targets **without the `--disable-jemalloc` option specified** all
targets will get jemalloc as they should.

Author: bors

Diff for: configure

@@ -865,9 +865,8 @@ fi
 # Force bitrig to build with clang; gcc doesn't like us there
 if [ $CFG_OSTYPE = unknown-bitrig ]
 then
-    step_msg "on Bitrig, forcing use of clang, disabling jemalloc"
+    step_msg "on Bitrig, forcing use of clang"
     CFG_ENABLE_CLANG=1
-    CFG_DISABLE_JEMALLOC=1
 fi
 
 # default gcc version under OpenBSD maybe too old, try using egcc, which is a
@@ -887,16 +886,6 @@ then
             CXX="${CXX:-eg++}"
         fi
     fi
-
-    step_msg "on OpenBSD, disabling jemalloc"
-    CFG_DISABLE_JEMALLOC=1
-fi
-
-if [ $CFG_OSTYPE = pc-windows-gnu ]
-then
-    # FIXME(#31030) - there's not a great reason to disable jemalloc here
-    step_msg "on Windows, disabling jemalloc"
-    CFG_DISABLE_JEMALLOC=1
 fi
 
 # OS X 10.9, gcc is actually clang. This can cause some confusion in the build
@@ -1186,12 +1175,6 @@ do
             ;;
 
         *-msvc)
-            # Currently the build system is not configured to build jemalloc
-            # with MSVC, so we omit this optional dependency.
-            step_msg "targeting MSVC, disabling jemalloc"
-            CFG_DISABLE_JEMALLOC=1
-            putvar CFG_DISABLE_JEMALLOC
-
             # There are some MSYS python builds which will auto-translate
             # windows-style paths to MSYS-style paths in Python itself.
             # Unfortunately this breaks LLVM's build system as somewhere along
@@ -1300,18 +1283,6 @@ $ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake
             putvar CFG_MSVC_LIB_PATH_${bits}
             ;;
 
-        *-rumprun-netbsd)
-            step_msg "targeting rumprun-netbsd, disabling jemalloc"
-            CFG_DISABLE_JEMALLOC=1
-            putvar CFG_DISABLE_JEMALLOC
-            ;;
-
-        *-emscripten)
-            step_msg "targeting emscripten, disabling jemalloc"
-            CFG_DISABLE_JEMALLOC=1
-            putvar CFG_DISABLE_JEMALLOC
-            ;;
-
         *)
             ;;
     esac

Diff for: mk/cfg/asmjs-unknown-emscripten.mk

@@ -21,3 +21,4 @@ CFG_LDPATH_asmjs-unknown-emscripten :=
 CFG_RUN_asmjs-unknown-emscripten=$(2)
 CFG_RUN_TARG_asmjs-unknown-emscripten=$(call CFG_RUN_asmjs-unknown-emscripten,,$(2))
 CFG_GNU_TRIPLE_asmjs-unknown-emscripten := asmjs-unknown-emscripten
+CFG_DISABLE_JEMALLOC_asmjs-unknown-emscripten := 1

Diff for: mk/cfg/i686-pc-windows-gnu.mk

@@ -25,3 +25,5 @@ CFG_GNU_TRIPLE_i686-pc-windows-gnu := i686-w64-mingw32
 CFG_THIRD_PARTY_OBJECTS_i686-pc-windows-gnu := crt2.o dllcrt2.o
 CFG_INSTALLED_OBJECTS_i686-pc-windows-gnu := crt2.o dllcrt2.o rsbegin.o rsend.o
 CFG_RUSTRT_HAS_STARTUP_OBJS_i686-pc-windows-gnu := 1
+# FIXME(#31030) - there's not a great reason to disable jemalloc here
+CFG_DISABLE_JEMALLOC_i686-pc-windows-gnu := 1

Diff for: mk/cfg/i686-pc-windows-msvc.mk

@@ -22,3 +22,7 @@ CFG_LDPATH_i686-pc-windows-msvc :=
 CFG_RUN_i686-pc-windows-msvc=$(2)
 CFG_RUN_TARG_i686-pc-windows-msvc=$(call CFG_RUN_i686-pc-windows-msvc,,$(2))
 CFG_GNU_TRIPLE_i686-pc-windows-msvc := i686-pc-win32
+
+# Currently the build system is not configured to build jemalloc
+# with MSVC, so we omit this optional dependency.
+CFG_DISABLE_JEMALLOC_i686-pc-windows-msvc := 1

Diff for: mk/cfg/x86_64-pc-windows-gnu.mk

@@ -25,3 +25,5 @@ CFG_GNU_TRIPLE_x86_64-pc-windows-gnu := x86_64-w64-mingw32
 CFG_THIRD_PARTY_OBJECTS_x86_64-pc-windows-gnu := crt2.o dllcrt2.o
 CFG_INSTALLED_OBJECTS_x86_64-pc-windows-gnu := crt2.o dllcrt2.o rsbegin.o rsend.o
 CFG_RUSTRT_HAS_STARTUP_OBJS_x86_64-pc-windows-gnu := 1
+# FIXME(#31030) - there's not a great reason to disable jemalloc here
+CFG_DISABLE_JEMALLOC_x86_64-pc-windows-gnu := 1

Diff for: mk/cfg/x86_64-pc-windows-msvc.mk

@@ -22,3 +22,7 @@ CFG_LDPATH_x86_64-pc-windows-msvc :=
 CFG_RUN_x86_64-pc-windows-msvc=$(2)
 CFG_RUN_TARG_x86_64-pc-windows-msvc=$(call CFG_RUN_x86_64-pc-windows-msvc,,$(2))
 CFG_GNU_TRIPLE_x86_64-pc-windows-msvc := x86_64-pc-win32
+
+# Currently the build system is not configured to build jemalloc
+# with MSVC, so we omit this optional dependency.
+CFG_DISABLE_JEMALLOC_x86_64-pc-windows-msvc := 1

Diff for: mk/cfg/x86_64-rumprun-netbsd.mk

@@ -22,3 +22,4 @@ CFG_LDPATH_x86_64-rumprun-netbsd :=
 CFG_RUN_x86_64-rumprun-netbsd=$(2)
 CFG_RUN_TARG_x86_64-rumprun-netbsd=$(call CFG_RUN_x86_64-rumprun-netbsd,,$(2))
 CFG_GNU_TRIPLE_x86_64-rumprun-netbsd := x86_64-rumprun-netbsd
+CFG_DISABLE_JEMALLOC_x86_64-rumprun-netbsd := 1

Diff for: mk/cfg/x86_64-unknown-bitrig.mk

@@ -20,3 +20,4 @@ CFG_LDPATH_x86_64-unknown-bitrig :=
 CFG_RUN_x86_64-unknown-bitrig=$(2)
 CFG_RUN_TARG_x86_64-unknown-bitrig=$(call CFG_RUN_x86_64-unknown-bitrig,,$(2))
 CFG_GNU_TRIPLE_x86_64-unknown-bitrig := x86_64-unknown-bitrig
+CFG_DISABLE_JEMALLOC_x86_64-unknown-bitrig := 1

Diff for: mk/cfg/x86_64-unknown-openbsd.mk

@@ -21,3 +21,4 @@ CFG_RUN_x86_64-unknown-openbsd=$(2)
 CFG_RUN_TARG_x86_64-unknown-openbsd=$(call CFG_RUN_x86_64-unknown-openbsd,,$(2))
 CFG_GNU_TRIPLE_x86_64-unknown-openbsd := x86_64-unknown-openbsd
 RUSTC_FLAGS_x86_64-unknown-openbsd=-C linker=$(call FIND_COMPILER,$(CC))
+CFG_DISABLE_JEMALLOC_x86_64-unknown-openbsd := 1

Diff for: mk/crates.mk

@@ -53,7 +53,7 @@ TARGET_CRATES := libc std flate arena term \
                  serialize getopts collections test rand \
                  log graphviz core rbml alloc \
                  rustc_unicode rustc_bitflags \
-		 alloc_system
+		 alloc_system alloc_jemalloc
 RUSTC_CRATES := rustc rustc_typeck rustc_mir rustc_borrowck rustc_resolve rustc_driver \
                 rustc_trans rustc_back rustc_llvm rustc_privacy rustc_lint \
                 rustc_data_structures rustc_front rustc_platform_intrinsics \
@@ -64,6 +64,7 @@ TOOLS := compiletest rustdoc rustc rustbook error_index_generator
 DEPS_core :=
 DEPS_alloc := core libc alloc_system
 DEPS_alloc_system := core libc
+DEPS_alloc_jemalloc := core libc native:jemalloc
 DEPS_collections := core alloc rustc_unicode
 DEPS_libc := core
 DEPS_rand := core
@@ -135,15 +136,14 @@ ONLY_RLIB_collections := 1
 ONLY_RLIB_rustc_unicode := 1
 ONLY_RLIB_rustc_bitflags := 1
 ONLY_RLIB_alloc_system := 1
+ONLY_RLIB_alloc_jemalloc := 1
+
+TARGET_SPECIFIC_alloc_jemalloc := 1
 
 # Documented-by-default crates
 DOC_CRATES := std alloc collections core libc rustc_unicode
 
 ifeq ($(CFG_DISABLE_JEMALLOC),)
-TARGET_CRATES += alloc_jemalloc
-DEPS_std += alloc_jemalloc
-DEPS_alloc_jemalloc := core libc native:jemalloc
-ONLY_RLIB_alloc_jemalloc := 1
 RUSTFLAGS_rustc_back := --cfg 'feature="jemalloc"'
 endif
 
@@ -160,12 +160,32 @@ CRATES := $(TARGET_CRATES) $(HOST_CRATES)
 define RUST_CRATE
 CRATEFILE_$(1) := $$(SREL)src/lib$(1)/lib.rs
 RSINPUTS_$(1) := $$(call rwildcard,$(S)src/lib$(1)/,*.rs)
-RUST_DEPS_$(1) := $$(filter-out native:%,$$(DEPS_$(1)))
 NATIVE_DEPS_$(1) := $$(patsubst native:%,%,$$(filter native:%,$$(DEPS_$(1))))
 endef
 
 $(foreach crate,$(CRATES),$(eval $(call RUST_CRATE,$(crate))))
 
+# $(1) - crate
+# $(2) - target
+define RUST_CRATE_DEPS
+RUST_DEPS_$(1)_T_$(2) := $$(filter-out native:%,$$(DEPS_$(1)))
+endef
+
+$(foreach target,$(CFG_TARGET),\
+ $(foreach crate,$(CRATES),$(eval $(call RUST_CRATE_DEPS,$(crate),$(target)))))
+
+# $(1) - target
+# $(2) - crate
+define DEFINE_TARGET_CRATES
+ifndef TARGET_SPECIFIC_$(2)
+TARGET_CRATES_$(1) += $(2)
+endif
+endef
+
+$(foreach target,$(CFG_TARGET),\
+ $(foreach crate,$(TARGET_CRATES),\
+  $(eval $(call DEFINE_TARGET_CRATES,$(target),$(crate)))))
+
 # Similar to the macro above for crates, this macro is for tools
 #
 # $(1) is the crate to generate variables for

Diff for: mk/docs.mk

@@ -158,9 +158,9 @@ LIB_DOC_DEP_$(1) = \
 	$$(CRATEFILE_$(1)) \
 	$$(RSINPUTS_$(1)) \
 	$$(RUSTDOC_EXE) \
-	$$(foreach dep,$$(RUST_DEPS_$(1)), \
+	$$(foreach dep,$$(RUST_DEPS_$(1)_T_$(CFG_BUILD)), \
 		$$(TLIB2_T_$(CFG_BUILD)_H_$(CFG_BUILD))/stamp.$$(dep)) \
-	$$(foreach dep,$$(filter $$(DOC_CRATES), $$(RUST_DEPS_$(1))), \
+	$$(foreach dep,$$(filter $$(DOC_CRATES), $$(RUST_DEPS_$(1)_T_$(CFG_BUILD))), \
 		doc/$$(dep)/)
 else
 LIB_DOC_DEP_$(1) = $$(CRATEFILE_$(1)) $$(RSINPUTS_$(1))

Diff for: mk/host.mk

@@ -21,7 +21,7 @@ define CP_HOST_STAGE_N_CRATE
 ifeq ($$(ONLY_RLIB_$(5)),)
 $$(HLIB$(2)_H_$(4))/stamp.$(5): \
 	$$(TLIB$(1)_T_$(3)_H_$(4))/stamp.$(5) \
-	$$(RUST_DEPS_$(5):%=$$(HLIB$(2)_H_$(4))/stamp.%) \
+	$$(RUST_DEPS_$(5)_T_$(3):%=$$(HLIB$(2)_H_$(4))/stamp.%) \
 	| $$(HLIB$(2)_H_$(4))/
 	@$$(call E, cp: $$(@D)/lib$(5))
 	$$(call REMOVE_ALL_OLD_GLOB_MATCHES, \

Diff for: mk/install.mk

@@ -108,15 +108,15 @@ endif
 define INSTALL_RUNTIME_TARGET_N
 install-runtime-target-$(1)-host-$(2): $$(TSREQ$$(ISTAGE)_T_$(1)_H_$(2)) $$(SREQ$$(ISTAGE)_T_$(1)_H_$(2))
 	$$(Q)$$(call ADB_SHELL,mkdir,$(CFG_RUNTIME_PUSH_DIR))
-	$$(Q)$$(foreach crate,$$(TARGET_CRATES), \
+	$$(Q)$$(foreach crate,$$(TARGET_CRATES_$(1)), \
 	    $$(call ADB_PUSH,$$(TL$(1)$(2))/$$(call CFG_LIB_GLOB_$(1),$$(crate)), \
 			$$(CFG_RUNTIME_PUSH_DIR));)
 endef
 
 define INSTALL_RUNTIME_TARGET_CLEANUP_N
 install-runtime-target-$(1)-cleanup:
 	$$(Q)$$(call ADB,remount)
-	$$(Q)$$(foreach crate,$$(TARGET_CRATES), \
+	$$(Q)$$(foreach crate,$$(TARGET_CRATES_$(1)), \
 	    $$(call ADB_SHELL,rm,$$(CFG_RUNTIME_PUSH_DIR)/$$(call CFG_LIB_GLOB_$(1),$$(crate)));)
 endef
 

Diff for: mk/main.mk

@@ -432,7 +432,7 @@ TSREQ$(1)_T_$(2)_H_$(3) = \
 # target
 SREQ$(1)_T_$(2)_H_$(3) = \
 	$$(TSREQ$(1)_T_$(2)_H_$(3)) \
-	$$(foreach dep,$$(TARGET_CRATES), \
+	$$(foreach dep,$$(TARGET_CRATES_$(2)), \
 	    $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(dep)) \
 	tmp/install-debugger-scripts$(1)_T_$(2)_H_$(3)-$$(call TRIPLE_TO_DEBUGGER_SCRIPT_SETTING,$(2)).done
 
@@ -441,7 +441,7 @@ SREQ$(1)_T_$(2)_H_$(3) = \
 CSREQ$(1)_T_$(2)_H_$(3) = \
 	$$(TSREQ$(1)_T_$(2)_H_$(3)) \
 	$$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
-	$$(foreach dep,$$(CRATES),$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(dep))
+	$$(foreach dep,$$(HOST_CRATES),$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(dep))
 
 ifeq ($(1),0)
 # Don't run the stage0 compiler under valgrind - that ship has sailed

Diff for: mk/platform.mk

@@ -118,6 +118,18 @@ endef
 $(foreach target,$(CFG_TARGET), \
   $(eval $(call DEFINE_LINKER,$(target))))
 
+define ADD_JEMALLOC_DEP
+  ifndef CFG_DISABLE_JEMALLOC_$(1)
+    ifndef CFG_DISABLE_JEMALLOC
+      RUST_DEPS_std_T_$(1) += alloc_jemalloc
+      TARGET_CRATES_$(1) += alloc_jemalloc
+    endif
+  endif
+endef
+
+$(foreach target,$(CFG_TARGET), \
+  $(eval $(call ADD_JEMALLOC_DEP,$(target))))
+
 # The -Qunused-arguments sidesteps spurious warnings from clang
 define FILTER_FLAGS
   ifeq ($$(CFG_USING_CLANG),1)

Diff for: mk/prepare.mk

@@ -122,7 +122,7 @@ prepare-host-lib-$(1)-$(2)-$(3)-$(4): \
 prepare-host-lib-$(1)-$(2)-$(3)-$(4): \
 	PREPARE_WORKING_DEST_LIB_DIR=$$(PREPARE_DEST_DIR)/$$(call PREPARE_TAR_LIB_DIR,$$(HLIB_RELATIVE$(2)_H_$(3)))
 prepare-host-lib-$(1)-$(2)-$(3)-$(4): prepare-maybe-clean-$(4) \
-                                 $$(foreach dep,$$(RUST_DEPS_$(1)),prepare-host-lib-$$(dep)-$(2)-$(3)-$(4)) \
+                                 $$(foreach dep,$$(RUST_DEPS_$(1)_T_$(3)),prepare-host-lib-$$(dep)-$(2)-$(3)-$(4)) \
                                  $$(HLIB$(2)_H_$(3))/stamp.$(1) \
                                  prepare-host-dirs-$(4)
 	$$(if $$(findstring $(2), $$(PREPARE_STAGE)), \
@@ -147,7 +147,7 @@ prepare-target-$(2)-host-$(3)-$(1)-$(4): \
 prepare-target-$(2)-host-$(3)-$(1)-$(4): \
 	PREPARE_DEST_BIN_DIR=$$(PREPARE_DEST_LIB_DIR)/rustlib/$(3)/bin
 prepare-target-$(2)-host-$(3)-$(1)-$(4): prepare-maybe-clean-$(4) \
-        $$(foreach crate,$$(TARGET_CRATES), \
+        $$(foreach crate,$$(TARGET_CRATES_$(2)), \
           $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(crate)) \
         $$(if $$(findstring $(2),$$(CFG_HOST)), \
           $$(foreach crate,$$(HOST_CRATES), \
@@ -161,7 +161,7 @@ prepare-target-$(2)-host-$(3)-$(1)-$(4): prepare-maybe-clean-$(4) \
         $$(if $$(findstring $(3), $$(PREPARE_HOST)), \
           $$(call PREPARE_DIR,$$(PREPARE_WORKING_DEST_LIB_DIR)) \
           $$(call PREPARE_DIR,$$(PREPARE_DEST_BIN_DIR)) \
-          $$(foreach crate,$$(TARGET_CRATES), \
+          $$(foreach crate,$$(TARGET_CRATES_$(2)), \
 	    $$(if $$(or $$(findstring 1, $$(ONLY_RLIB_$$(crate))),$$(findstring 1,$$(CFG_INSTALL_ONLY_RLIB_$(2)))),, \
               $$(call PREPARE_LIB,$$(call CFG_LIB_GLOB_$(2),$$(crate)))) \
             $$(call PREPARE_LIB,$$(call CFG_RLIB_GLOB,$$(crate)))) \

Diff for: mk/target.mk

@@ -28,7 +28,7 @@ define RUST_CRATE_FULLDEPS
 CRATE_FULLDEPS_$(1)_T_$(2)_H_$(3)_$(4) := \
 		$$(CRATEFILE_$(4)) \
 		$$(RSINPUTS_$(4)) \
-		$$(foreach dep,$$(RUST_DEPS_$(4)), \
+		$$(foreach dep,$$(RUST_DEPS_$(4)_T_$(2)), \
 		  $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(dep)) \
 		$$(foreach dep,$$(NATIVE_DEPS_$(4)), \
 		  $$(RT_OUTPUT_DIR_$(2))/$$(call CFG_STATIC_LIB_NAME_$(2),$$(dep))) \
@@ -147,7 +147,7 @@ ifeq ($$(CFG_RUSTRT_HAS_STARTUP_OBJS_$(2)), 1)
 # Add dependencies on Rust startup objects to all crates that depend on core.
 # This ensures that they are built after core (since they depend on it),
 # but before everything else (since they are needed for linking dylib crates).
-$$(foreach crate, $$(TARGET_CRATES), \
+$$(foreach crate, $$(TARGET_CRATES_$(2)), \
 	$$(if $$(findstring core,$$(DEPS_$$(crate))), \
 		$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(crate))) : $$(TLIB$(1)_T_$(2)_H_$(3))/$(4).o
 endif

Diff for: mk/tests.mk

@@ -135,7 +135,7 @@ $(info check: android device test dir $(CFG_ADB_TEST_DIR) ready \
  $(foreach target,$(CFG_TARGET), \
   $(if $(findstring android, $(target)), \
    $(shell $(CFG_ADB) shell mkdir $(CFG_ADB_TEST_DIR)/$(target)) \
-   $(foreach crate,$(TARGET_CRATES), \
+   $(foreach crate,$(TARGET_CRATES_$(target)), \
     $(shell $(CFG_ADB) push $(TLIB2_T_$(target)_H_$(CFG_BUILD))/$(call CFG_LIB_GLOB_$(target),$(crate)) \
                     $(CFG_ADB_TEST_DIR)/$(target))), \
  )))
@@ -364,7 +364,7 @@ define TEST_RUNNER
 # parent crates.
 ifeq ($(NO_REBUILD),)
 TESTDEP_$(1)_$(2)_$(3)_$(4) = $$(SREQ$(1)_T_$(2)_H_$(3)) \
-			    $$(foreach crate,$$(TARGET_CRATES), \
+			    $$(foreach crate,$$(TARGET_CRATES_$(2)), \
 				$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$$(crate)) \
 				$$(CRATE_FULLDEPS_$(1)_T_$(2)_H_$(3)_$(4))