Fix executable and more library RPATHs on FreeBSD (#27627)

First of all, we need to add libgfortran, libgcc_s, and libquadmath to
`JL_PRIVATE_LIBS-0` instead of `JL_LIBS`. This ensures that they get
installed into `lib/julia`, where our dependencies are supposed to live,
instead of `lib`, where only libjulia should live.

Now we need to call `fixup-rpath.sh` on more directories. Currently
we're only calling it on `lib`, but we also need to call it on
`lib/julia` and even on `bin`, because FreeBSD seems to load libraries
based on the executable's RPATH rather than libjulia's RPATH.

AND, speaking of the executable's RPATH, we need to make sure we have
`lib/julia` in there; currently we only have `lib`. So in an isolated
environment outside of the build directory, Julia won't be able to load
any of its dependencies, which is bad. This is accomplished with a
simple adjustment in `Make.inc`.

To make sure that our dependencies can find each other properly, we need
to add `RPATH_ESCAPED_ORIGIN` to `LDFLAGS` in `CONFIGURE_COMMON`. That
way, every dependency that uses `configure` will get its RPATH set
appropriately.

(cherry picked from commit 07edead)

Make.inc

@@ -934,6 +934,9 @@ else ifeq ($(OS), Darwin)
   RPATH_LIB := -Wl,-rpath,'@loader_path/julia/' -Wl,-rpath,'@loader_path/'
 else
   RPATH := -Wl,-rpath,'$$ORIGIN/$(build_libdir_rel)' -Wl,-rpath-link,$(build_shlibdir) -Wl,-z,origin
+ifeq ($(OS), FreeBSD)
+  RPATH += -Wl,-rpath,'$$ORIGIN/$(build_private_libdir_rel)'
+endif
   RPATH_ORIGIN := -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin
   RPATH_ESCAPED_ORIGIN := -Wl,-rpath,'\$$\$$ORIGIN' -Wl,-z,origin
   RPATH_LIB := -Wl,-rpath,'$$ORIGIN/julia' -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin

Makefile

@@ -309,17 +309,22 @@ endif
 endif
 endif
 
+# On FreeBSD, /lib/libgcc_s.so.1 is incompatible with Fortran; to use Fortran on FreeBSD,
+# we need to link to the libgcc_s that ships with the same GCC version used by libgfortran.
+# To work around this, we copy the GCC libraries we need, namely libgfortran, libgcc_s,
+# and libquadmath, into our build library directory, $(build_libdir). We also add them to
+# JL_PRIVATE_LIBS-0 so that they know where they need to live at install time.
 ifeq ($(OS),FreeBSD)
 define std_so
-julia-deps: | $$(build_libdir)/lib$(1).so
-$$(build_libdir)/lib$(1).so: | $$(build_libdir)
-	$$(INSTALL_M) $$(GCCPATH)/lib$(1).so* $$(build_libdir)
-JL_LIBS += $(1)
+julia-deps: | $$(build_libdir)/$(1).so
+$$(build_libdir)/$(1).so: | $$(build_libdir)
+	$$(INSTALL_M) $$(GCCPATH)/$(1).so* $$(build_libdir)
+JL_PRIVATE_LIBS-0 += $(1)
 endef
 
-$(eval $(call std_so,gfortran))
-$(eval $(call std_so,gcc_s))
-$(eval $(call std_so,quadmath))
+$(eval $(call std_so,libgfortran))
+$(eval $(call std_so,libgcc_s))
+$(eval $(call std_so,libquadmath))
 endif # FreeBSD
 
 ifeq ($(OS),WINNT)
@@ -433,6 +438,15 @@ endif
 	# On FreeBSD, remove the build's libdir from each library's RPATH
 ifeq ($(OS),FreeBSD)
 	$(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(libdir) $(build_libdir)
+	$(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(private_libdir) $(build_libdir)
+	$(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(bindir) $(build_libdir)
+	# Set libgfortran's RPATH to ORIGIN instead of GCCPATH. It's only libgfortran that
+	# needs to be fixed here, as libgcc_s and libquadmath don't have RPATHs set. If we
+	# don't set libgfortran's RPATH, it won't be able to find its friends on systems
+	# that don't have the exact GCC port installed used for the build.
+	for lib in $(DESTDIR)$(private_libdir)/libgfortran*$(SHLIB_EXT)*; do \
+		$(build_depsbindir)/patchelf --set-rpath '$$ORIGIN' $$lib; \
+	done
 endif
 
 	mkdir -p $(DESTDIR)$(sysconfdir)
@@ -461,7 +475,11 @@ ifneq ($(DESTDIR),)
 endif
 	@$(MAKE) -C $(BUILDROOT) -f $(JULIAHOME)/Makefile install
 	cp $(JULIAHOME)/LICENSE.md $(BUILDROOT)/julia-$(JULIA_COMMIT)
-ifneq ($(OS), WINNT)
+	# Run fixup-libgfortran on all platforms but Windows and FreeBSD. On FreeBSD we
+	# pull in the GCC libraries earlier and use them for the build to make sure we
+	# don't inadvertently link to /lib/libgcc_s.so.1, which is incompatible with
+	# libgfortran.
+ifeq (,$(findstring $(OS),FreeBSD WINNT))
 	-$(CUSTOM_LD_LIBRARY_PATH) PATH=$(PATH):$(build_depsbindir) $(JULIAHOME)/contrib/fixup-libgfortran.sh $(DESTDIR)$(private_libdir)
 endif
 ifeq ($(OS), Linux)

contrib/fixup-rpath.sh

@@ -1,17 +1,17 @@
 #!/bin/sh
-# Usage: fixup-rpath.sh <patchelf path> <install libdir> <build libdir>
+# Usage: fixup-rpath.sh <patchelf path> <dir to process> <build libdir>
 
 if [ $# -ne 3 ]; then
     echo "Incorrect number of arguments: Expected 3, got $#"
-    echo "Usage: fixup-rpath.sh <patchelf path> <install libdir> <build libdir>"
+    echo "Usage: fixup-rpath.sh <patchelf path> <directory to process> <build libdir>"
     exit 1
 fi
 
 patchelf="$1"
-install_libdir="$2"
+executable_dir="$2"
 build_libdir="$3"
 
-for lib in ${install_libdir}/*.so*; do
+for lib in $(find ${executable_dir} -type f -perm -111); do
     # First get the current RPATH
     rpath="$(${patchelf} --print-rpath ${lib})"
 
@@ -25,6 +25,8 @@ for lib in ${install_libdir}/*.so*; do
     # Drop the trailing :
     new_rpath="${new_rpath%?}"
 
+    echo "  Setting RPATH for ${lib} to '${new_rpath}'"
+
     # Now set the new RPATH
     ${patchelf} --set-rpath "${new_rpath}" ${lib}
 done

deps/tools/common.mk

@@ -12,6 +12,8 @@ ifeq ($(OS),WINNT)
 ifneq ($(USEMSVC), 1)
 CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) -Wl,--stack,8388608"
 endif
+else
+CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN)"
 endif
 CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(DEPS_CFLAGS)" CXX="$(CXX) $(DEPS_CXXFLAGS)"