[beta] FreeBSD beta fixes

[bdrewery]

r? @alexcrichton
r? @Mark-Simulacrum

[rust-highfive]

Thanks for the pull request, and welcome! The Rust team is excited to review your changes, and you should hear from @alexcrichton (or someone else) soon.

If any changes to this PR are deemed necessary, please add them as extra commits. This ensures that the reviewer can see what has changed since they last reviewed the code. Due to the way GitHub handles out-of-date commits, this should also make it reasonably obvious what issues have or haven't been addressed. Large or tricky changes may require several passes of review and changes.

Please see the contribution instructions for more information.

[alexcrichton]

Thanks for the PR @bdrewery! We're broadly try to be conservative about backporting to the beta branch, so mind detailing a bit as to the rationale for the backport? (mostly so we've got a record)

Overall these changes all are harmless/small enough to me that I think it should be fine to backport!

[bdrewery]

1. On FreeBSD 12 it is impossible to build from git without hacking at the bootstrap that is downloaded since it is using the wrong fstat symbols (fs::metadata() crashes on FreeBSD 12 due to layout change in stat.h #42681). Currently I'm using this script which also depends on having a ports tree checked out (which is probably not needed) but the patching of libstd to fix fstat symbols is needed:

#! /bin/sh

PORT="${PORTSDIR}/lang/rust-nightly"
OSVERSION=$(awk '/\#define __FreeBSD_version/ { print $3 }' "/usr/include/sys/param.h")
BOOTSTRAPS_DATE="$(make -C ${PORT} -V BOOTSTRAPS_DATE)" RUSTC_BOOTSTRAP="$(make -C ${PORT} -V RUSTC_BOOTSTRAP)"
FILESDIR="$(make -C ${PORT} -V FILESDIR)"
RUST_STD_BOOTSTRAP="$(make -C ${PORT} -V RUST_STD_BOOTSTRAP)"
CARGO_BOOTSTRAP="$(make -C ${PORT} -V CARGO_BOOTSTRAP)"
RUST_TARGET="$(make -C ${PORT} -V RUST_TARGET)"
DISTDIR="$(make -C ${PORT} -V '${DISTDIR}/${DIST_SUBDIR}')"
rm -rf build
mkdir -p "build/cache/${BOOTSTRAPS_DATE}"
ln -sf "${DISTDIR}/${RUSTC_BOOTSTRAP}" "build/cache/${BOOTSTRAPS_DATE}"
ln -sf "${DISTDIR}/${CARGO_BOOTSTRAP}" "build/cache/${BOOTSTRAPS_DATE}"
if [ ${OSVERSION} -lt 1200031 ]; then
        ln -sf "${DISTDIR}/${RUST_STD_BOOTSTRAP}" "build/cache/${BOOTSTRAPS_DATE}"
else
        TMP=$(mktemp -d)
        (
                set -ex
                cd "${TMP}"
                cc -O2 -pipe  -fstack-protector -fno-strict-aliasing \
                    -fPIC -c -o old_fstat.o "${FILESDIR}/old_fstat.c"
                tar -xf "${DISTDIR}/${RUST_STD_BOOTSTRAP}"
                RUST_STD_DIR="${RUST_STD_BOOTSTRAP##*/}"
                RUST_STD_DIR="${RUST_STD_DIR%%.*}"
                libstd="$(echo "${RUST_STD_DIR}/rust-std-${RUST_TARGET}/lib/rustlib/${RUST_TARGET}/lib/"libstd-*.rlib)"
                echo "libstd='${libstd}'"
                hash="${libstd##*/libstd-}"
                hash="${hash%.rlib}"
                std_o="$(ar t "${libstd}" | grep -E "^std-${hash}.*\.o$" | head -n 1)"
                ar x "${libstd}" "${std_o}"
                ld -r -o std.xx.o "${std_o}" old_fstat.o
                mv -f std.xx.o "${std_o}"
                ar r "${libstd}" "${std_o}"
                tar -c --format=ustar -f "${RUST_STD_BOOTSTRAP##*/}" \
                    "${RUST_STD_DIR}"
                mv -f "${RUST_STD_BOOTSTRAP##*/}" \
                    "${OLDPWD}/build/cache/${BOOTSTRAPS_DATE}/"
        )
        rm -rf "${TMP}"
fi

The old_fstat.c file is:

/* $FreeBSD: head/lang/rust/files/old_fstat.c 441843 2017-05-27 12:06:40Z kib $ */

#include <sys/syscall.h>
#include <dlfcn.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

struct stat;

int
fstat(int fd, struct stat *sb)
{

	return (syscall(SYS_freebsd11_fstat, fd, sb));
}

int
stat(const char *path, struct stat *sb)
{

	return (syscall(SYS_freebsd11_stat, path, sb));
}

int
lstat(const char *path, struct stat *sb)
{

	return (syscall(SYS_freebsd11_lstat, path, sb));
}

int
fstatat(int fd, const char *path, struct stat *sb, int flag)
{

	return (syscall(SYS_freebsd11_fstatat, fd, path, sb, flag));
}

struct ODIR;
struct freebsd11_dirent;

/*
__asm(".symver  old_readdir, readdir@FBSD_1.0");
int old_readdir(struct ODIR *);
int
readdir(struct ODIR *dirp)
{

	return (old_readdir(dirp));
}
*/

__asm(".symver  old_readdir_r, readdir_r@FBSD_1.0");
int old_readdir_r(struct ODIR *, struct freebsd11_dirent *,
    struct freebsd11_dirent **);
int
readdir_r(struct ODIR *dirp, struct freebsd11_dirent *entry,
    struct freebsd11_dirent **result)
{
	void *libc;
	void *fptr;
	int error;

	libc = dlopen("libc.so.7", RTLD_LAZY | RTLD_GLOBAL);
	if (libc == NULL) {
		fprintf(stderr, "libc open: %s\n", dlerror());
		abort();
	}
	fptr = dlvsym(libc, "readdir_r", "FBSD_1.0");
	if (fptr == NULL) {
		fprintf(stderr, "readdir_r: %s\n", dlerror());
		abort();
	}
	error = ((int (*)(struct ODIR *, struct freebsd11_dirent *,
	    struct freebsd11_dirent **))fptr)(dirp, entry, result);
	dlclose(libc);
	return (error);
}

/*
__asm(".symver	old_scandir, scandir@FBSD_1.0");
int old_scandir(const char *, struct freebsd11_dirent ***,
    int (*)(const struct freebsd11_dirent *),
    int (*)(const struct freebsd11_dirent **,
	const struct freebsd11_dirent **));
int
scandir(const char *dirname, struct freebsd11_dirent ***namelist,
    int (*select)(const struct freebsd11_dirent *),
    int (*dcomp)(const struct freebsd11_dirent **,
	const struct freebsd11_dirent **))
{

	return (old_scandir(dirname, namelist, select, dcomp));
}
*/

struct old_statfs;
int
fstatfs(int fd, struct old_statfs *buf)
{

	return (syscall(SYS_freebsd11_fstatfs, fd, buf));
}

After running this the build can proceed. We also have the same hack in the official Port build as well. Any kind of patching like this is nasty and we'd like to remove the need for it. It's not documented anywhere either and has caused grief for rust development on FreeBSD 12 as seen in the linked bug.

(Unrelated to the beta, similar patching was needed to ~/.cargo/registry's libc and the libc in the checkout before libc was updated in master to bring in rust-lang/libc@78f9322)

Without this patching the build simply fails in an obscure way very early:

     Running `/root/git/rust/rust2/build/x86_64-unknown-freebsd/stage0/bin/rustc --crate-name bootstrap bootstrap/bin/main.rs --crate-type bin --emit=dep-info,link -C debug-assertions=off -C overflow-checks=on -C metadata=6a809e7c34284e41 -C extra-filename=-6a809e7c34284e41 --out-d
ir /root/git/rust/rust2/build/bootstrap/debug/deps -C incremental=/root/git/rust/rust2/build/bootstrap/debug/incremental -L dependency=/root/git/rust/rust2/build/bootstrap/debug/deps --extern time=/root/git/rust/rust2/build/bootstrap/debug/deps/libtime-3b66b75744ce2323.rlib --exter
n serde_derive=/root/git/rust/rust2/build/bootstrap/debug/deps/libserde_derive-cb886017e511511e.so --extern build_helper=/root/git/rust/rust2/build/bootstrap/debug/deps/libbuild_helper-16ab4b07b9953b2a.rlib --extern getopts=/root/git/rust/rust2/build/bootstrap/debug/deps/libgetopts
-36a890615bb502e9.rlib --extern serde_json=/root/git/rust/rust2/build/bootstrap/debug/deps/libserde_json-c9a6ee565ee8f059.rlib --extern lazy_static=/root/git/rust/rust2/build/bootstrap/debug/deps/liblazy_static-0b94e2d6af7f839e.rlib --extern toml=/root/git/rust/rust2/build/bootstra
p/debug/deps/libtoml-76752341452dc1e9.rlib --extern cmake=/root/git/rust/rust2/build/bootstrap/debug/deps/libcmake-9e8025efc41ffe3d.rlib --extern num_cpus=/root/git/rust/rust2/build/bootstrap/debug/deps/libnum_cpus-f673db66666375f8.rlib --extern libc=/root/git/rust/rust2/build/boot
strap/debug/deps/liblibc-6335dae651da4b2d.rlib --extern filetime=/root/git/rust/rust2/build/bootstrap/debug/deps/libfiletime-d6f893a57ef1c5fb.rlib --extern serde=/root/git/rust/rust2/build/bootstrap/debug/deps/libserde-b01b65b0fba78017.rlib --extern cc=/root/git/rust/rust2/build/bo
otstrap/debug/deps/libcc-823736f6ee40bca8.rlib --extern bootstrap=/root/git/rust/rust2/build/bootstrap/debug/deps/libbootstrap-064ed40d3714f7cc.rlib -Cdebuginfo=2`
    Finished dev [unoptimized] target(s) in 64.89 secs
running: /root/git/rust/rust2/build/bootstrap/debug/bootstrap build --jobs 16 --verbose
Traceback (most recent call last):
  File "./x.py", line 20, in <module>
    bootstrap.main()
  File "/root/git/rust/rust2/src/bootstrap/bootstrap.py", line 763, in main
    bootstrap()
  File "/root/git/rust/rust2/src/bootstrap/bootstrap.py", line 754, in bootstrap
    run(args, env=env, verbose=build.verbose)
  File "/root/git/rust/rust2/src/bootstrap/bootstrap.py", line 148, in run
    raise RuntimeError(err)
RuntimeError: failed to run: /root/git/rust/rust2/build/bootstrap/debug/bootstrap build --jobs 16 --verbose
      103.00 real       172.49 user        28.16 sys

2. The 2nd commit for Workaround abort(2) on compilation error on FreeBSD. #48494 is fixing the bootstrap rustc aborting from errors:

# build/x86_64-unknown-freebsd/stage0/bin/rustc -v
error: no input filename given

fatal runtime error: failed to initiate panic, error 5
zsh: abort (core dumped)  build/x86_64-unknown-freebsd/stage0/bin/rustc -v

Vs the expected:

# rustc -v
error: no input filename given

I'm not sure how this abort manifests in early compile failures but it would be nice to not be distracted by it.

[bdrewery]

About (1) it may have been unclear but the main commit needed in libc is rust-lang/libc@78f9322.

[alexcrichton]

@bors: r+

Awesome, thanks for writing that out @bdrewery!

[bors]

📌 Commit dad5755 has been approved by alexcrichton

[bdrewery]

What's the next step for src/stage0.txt in master? Should I just wait for a release to require updating it or can we update it after this PR?

[alexcrichton]

Indeed yeah once this is merged a new beta will be produced, and then we can update src/stage0.txt on master to bootstrap from the new beta

[bdrewery]

Great!

[kennytm]

@bors p=2

[bors]

⌛ Testing commit dad5755 with merge acbd433...

[bors]

☀️ Test successful - status-appveyor, status-travis
Approved by: alexcrichton
Pushing acbd433 to beta...