Auto merge of #32942 - alexcrichton:bootstrap-from-previous, r=brson

mk: Bootstrap from stable instead of snapshots

This commit removes all infrastructure from the repository for our so-called
snapshots to instead bootstrap the compiler from stable releases. Bootstrapping
from a previously stable release is a long-desired feature of distros because
they're not fans of downloading binary stage0 blobs from us. Additionally, this
makes our own CI easier as we can decommission all of the snapshot builders and
start having a regular cadence to when we update the stage0 compiler.

A new `src/etc/get-stage0.py` script was added which shares some code with
`src/bootstrap/bootstrap.py` to read a new file, `src/stage0.txt`, which lists
the current stage0 compiler as well as cargo that we bootstrap from. This script
will download the relevant `rustc` package an unpack it into `$target/stage0` as
we do today.

One problem of bootstrapping from stable releases is that we're not able to
compile unstable code (e.g. all the `#![feature]` directives in libcore/libstd).
To overcome this we employ two strategies:

* The bootstrap key of the previous compiler is hardcoded into `src/stage0.txt`
  (enabled as a result of #32731) and exported by the build system. This enables
  nightly features in the compiler we download.
* The standard library and compiler are pinned to a specific stage0, which
  doesn't change, so we're guaranteed that we'll continue compiling as we start
  from a known fixed source.

The process for making a release will also need to be tweaked now to continue to
cadence of bootstrapping from the previous release. This process looks like:

1. Merge `beta` to `stable`
2. Produce a new stable compiler.
3. Change `master` to bootstrap from this new stable compiler.
4. Merge `master` to `beta`
5. Produce a new beta compiler
6. Change `master` to bootstrap from this new beta compiler.

Step 3 above should involve very few changes as `master` was previously
bootstrapping from `beta` which is the same as `stable` at that point in time.
Step 6, however, is where we benefit from removing lots of `#[cfg(stage0)]` and
get to use new features. This also shouldn't slow the release too much as steps
1-5 requires little work other than waiting and step 6 just needs to happen at
some point during a release cycle, it's not time sensitive.

Closes #29555
Closes #29557

Makefile.in

@@ -214,13 +214,6 @@ include $(CFG_SRC_DIR)mk/debuggers.mk
 # Secondary makefiles, conditionalized for speed
 ######################################################################
 
-# Binary snapshots
-ifneq ($(strip $(findstring snap,$(MAKECMDGOALS))   \
-               $(findstring clean,$(MAKECMDGOALS))),)
-  CFG_INFO := $(info cfg: including snap rules)
-  include $(CFG_SRC_DIR)mk/snap.mk
-endif
-
 # The test suite
 ifneq ($(strip $(findstring check,$(MAKECMDGOALS)) \
                $(findstring test,$(MAKECMDGOALS))  \

mk/dist.mk

@@ -61,7 +61,7 @@ PKG_FILES := \
       rtstartup                                \
       rustllvm                                 \
       rustc                                    \
-      snapshots.txt                            \
+      stage0.txt                               \
       rust-installer                           \
       tools                                    \
       test)                                    \

mk/main.mk

@@ -34,6 +34,7 @@ CFG_FILENAME_EXTRA=$(shell printf '%s' $(CFG_RELEASE)$(CFG_EXTRA_FILENAME) | $(C
 # intentionally not "secure" by any definition, this is largely just a deterrent
 # from users enabling unstable features on the stable compiler.
 CFG_BOOTSTRAP_KEY=$(CFG_FILENAME_EXTRA)
+CFG_BOOTSTRAP_KEY_STAGE0=$(shell grep 'rustc_key' $(S)src/stage0.txt | sed 's/rustc_key: '//)
 
 ifeq ($(CFG_RELEASE_CHANNEL),stable)
 # This is the normal semver version string, e.g. "0.12.0", "0.12.0-nightly"

mk/reconfig.mk

@@ -38,6 +38,6 @@ else
 SREL_ROOT := $(SREL)
 endif
 
-config.stamp: $(S)configure $(S)Makefile.in $(S)src/snapshots.txt
+config.stamp: $(S)configure $(S)Makefile.in $(S)src/stage0.txt
 	@$(call E, cfg: reconfiguring)
 	$(SREL_ROOT)configure $(CFG_CONFIGURE_ARGS)

mk/stage0.mk

@@ -10,17 +10,14 @@ $(HLIB0_H_$(CFG_BUILD))/:
 endif
 
 $(SNAPSHOT_RUSTC_POST_CLEANUP): \
-		$(S)src/snapshots.txt \
-		$(S)src/etc/get-snapshot.py $(MKFILE_DEPS) \
+		$(S)src/stage0.txt \
+		$(S)src/etc/get-stage0.py $(MKFILE_DEPS) \
 		| $(HBIN0_H_$(CFG_BUILD))/
-
 	@$(call E, fetch: $@)
-#   Note: the variable "SNAPSHOT_FILE" is generally not set, and so
-#   we generally only pass one argument to this script.
 ifdef CFG_ENABLE_LOCAL_RUST
 	$(Q)$(S)src/etc/local_stage0.sh $(CFG_BUILD) $(CFG_LOCAL_RUST_ROOT) rustlib
 else
-	$(Q)$(CFG_PYTHON) $(S)src/etc/get-snapshot.py $(CFG_BUILD) $(SNAPSHOT_FILE)
+	$(Q)$(CFG_PYTHON) $(S)src/etc/get-stage0.py $(CFG_BUILD)
 endif
 	$(Q)if [ -e "$@" ]; then touch "$@"; else echo "ERROR: snapshot $@ not found"; exit 1; fi
 

mk/target.mk

@@ -65,6 +65,11 @@ $(foreach host,$(CFG_HOST), \
 # $(4) is the crate name
 define RUST_TARGET_STAGE_N
 
+ifeq ($(1),0)
+$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
+	export RUSTC_BOOTSTRAP_KEY := $$(CFG_BOOTSTRAP_KEY_STAGE0)
+endif
+
 $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): CFG_COMPILER_HOST_TRIPLE = $(2)
 $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
 		$$(CRATEFILE_$(4)) \
@@ -113,6 +118,11 @@ endef
 # $(4) - name of the tool being built
 define TARGET_TOOL
 
+ifeq ($(1),0)
+$$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
+	export RUSTC_BOOTSTRAP_KEY := $$(CFG_BOOTSTRAP_KEY_STAGE0)
+endif
+
 $$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
 		$$(TOOL_SOURCE_$(4)) \
 		$$(TOOL_INPUTS_$(4)) \
@@ -167,11 +177,15 @@ SNAPSHOT_RUSTC_POST_CLEANUP=$(HBIN0_H_$(CFG_BUILD))/rustc$(X_$(CFG_BUILD))
 
 define TARGET_HOST_RULES
 
-$$(TLIB$(1)_T_$(2)_H_$(3))/:
+$$(TLIB$(1)_T_$(2)_H_$(3))/: $$(SNAPSHOT_RUSTC_POST_CLEANUP)
+	mkdir -p $$@
+
+$$(TBIN$(1)_T_$(2)_H_$(3))/: $$(SNAPSHOT_RUSTC_POST_CLEANUP)
 	mkdir -p $$@
 
 $$(TLIB$(1)_T_$(2)_H_$(3))/%: $$(RT_OUTPUT_DIR_$(2))/% \
-	    | $$(TLIB$(1)_T_$(2)_H_$(3))/ $$(SNAPSHOT_RUSTC_POST_CLEANUP)
+	    $$(SNAPSHOT_RUSTC_POST_CLEANUP) \
+	    | $$(TLIB$(1)_T_$(2)_H_$(3))/
 	@$$(call E, cp: $$@)
 	$$(Q)cp $$< $$@
 endef

mk/tests.mk

@@ -241,13 +241,16 @@ cleantestlibs:
 ######################################################################
 
 .PHONY: tidy
-tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD))
+tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)) \
+		$(SNAPSHOT_RUSTC_POST_CLEANUP)
 	$(TARGET_RPATH_VAR0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $< $(S)src
 
 $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)): \
 		$(TSREQ0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) \
 		$(TLIB0_T_$(CFG_BUILD)_H_$(CFG_BUILD))/stamp.std \
-		$(call rwildcard,$(S)src/tools/tidy/src,*.rs)
+		$(call rwildcard,$(S)src/tools/tidy/src,*.rs) \
+		$(SNAPSHOT_RUSTC_POST_CLEANUP) | \
+		$(TLIB0_T_$(CFG_BUILD)_H_$(CFG_BUILD))
 	$(STAGE0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $(S)src/tools/tidy/src/main.rs \
 		--out-dir $(@D) --crate-name tidy
 

src/bootstrap/bootstrap.py

@@ -79,11 +79,22 @@ def run(args, verbose=False):
             raise RuntimeError(err)
         sys.exit(err)
 
+def stage0_data(rust_root):
+    nightlies = os.path.join(rust_root, "src/stage0.txt")
+    with open(nightlies, 'r') as nightlies:
+        data = {}
+        for line in nightlies.read().split("\n"):
+            if line.startswith("#") or line == '':
+                continue
+            a, b = line.split(": ", 1)
+            data[a] = b
+        return data
+
 class RustBuild:
-    def download_rust_nightly(self):
+    def download_stage0(self):
         cache_dst = os.path.join(self.build_dir, "cache")
-        rustc_cache = os.path.join(cache_dst, self.snap_rustc_date())
-        cargo_cache = os.path.join(cache_dst, self.snap_cargo_date())
+        rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
+        cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
         if not os.path.exists(rustc_cache):
             os.makedirs(rustc_cache)
         if not os.path.exists(cargo_cache):
@@ -93,41 +104,49 @@ def download_rust_nightly(self):
            (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
             if os.path.exists(self.bin_root()):
                 shutil.rmtree(self.bin_root())
-            filename = "rust-std-nightly-" + self.build + ".tar.gz"
-            url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
+            channel = self.stage0_rustc_channel()
+            filename = "rust-std-" + channel + "-" + self.build + ".tar.gz"
+            url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
             tarball = os.path.join(rustc_cache, filename)
             if not os.path.exists(tarball):
                 get(url + "/" + filename, tarball, verbose=self.verbose)
             unpack(tarball, self.bin_root(),
                    match="rust-std-" + self.build,
                    verbose=self.verbose)
 
-            filename = "rustc-nightly-" + self.build + ".tar.gz"
-            url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
+            filename = "rustc-" + channel + "-" + self.build + ".tar.gz"
+            url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
             tarball = os.path.join(rustc_cache, filename)
             if not os.path.exists(tarball):
                 get(url + "/" + filename, tarball, verbose=self.verbose)
             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
             with open(self.rustc_stamp(), 'w') as f:
-                f.write(self.snap_rustc_date())
+                f.write(self.stage0_rustc_date())
 
         if self.cargo().startswith(self.bin_root()) and \
            (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
-            filename = "cargo-nightly-" + self.build + ".tar.gz"
-            url = "https://static.rust-lang.org/cargo-dist/" + self.snap_cargo_date()
+            channel = self.stage0_cargo_channel()
+            filename = "cargo-" + channel + "-" + self.build + ".tar.gz"
+            url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date()
             tarball = os.path.join(cargo_cache, filename)
             if not os.path.exists(tarball):
                 get(url + "/" + filename, tarball, verbose=self.verbose)
             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
             with open(self.cargo_stamp(), 'w') as f:
-                f.write(self.snap_cargo_date())
+                f.write(self.stage0_cargo_date())
 
-    def snap_cargo_date(self):
+    def stage0_cargo_date(self):
         return self._cargo_date
 
-    def snap_rustc_date(self):
+    def stage0_cargo_channel(self):
+        return self._cargo_channel
+
+    def stage0_rustc_date(self):
         return self._rustc_date
 
+    def stage0_rustc_channel(self):
+        return self._rustc_channel
+
     def rustc_stamp(self):
         return os.path.join(self.bin_root(), '.rustc-stamp')
 
@@ -138,13 +157,13 @@ def rustc_out_of_date(self):
         if not os.path.exists(self.rustc_stamp()):
             return True
         with open(self.rustc_stamp(), 'r') as f:
-            return self.snap_rustc_date() != f.read()
+            return self.stage0_rustc_date() != f.read()
 
     def cargo_out_of_date(self):
         if not os.path.exists(self.cargo_stamp()):
             return True
         with open(self.cargo_stamp(), 'r') as f:
-            return self.snap_cargo_date() != f.read()
+            return self.stage0_cargo_date() != f.read()
 
     def bin_root(self):
         return os.path.join(self.build_dir, self.build, "stage0")
@@ -187,15 +206,6 @@ def exe_suffix(self):
         else:
             return ''
 
-    def parse_nightly_dates(self):
-        nightlies = os.path.join(self.rust_root, "src/nightlies.txt")
-        with open(nightlies, 'r') as nightlies:
-            rustc, cargo = nightlies.read().split("\n")[:2]
-            assert rustc.startswith("rustc: ")
-            assert cargo.startswith("cargo: ")
-            self._rustc_date = rustc[len("rustc: "):]
-            self._cargo_date = cargo[len("cargo: "):]
-
     def build_bootstrap(self):
         env = os.environ.copy()
         env["CARGO_TARGET_DIR"] = os.path.join(self.build_dir, "bootstrap")
@@ -300,46 +310,53 @@ def build_triple(self):
 
         return cputype + '-' + ostype
 
-parser = argparse.ArgumentParser(description='Build rust')
-parser.add_argument('--config')
-parser.add_argument('-v', '--verbose', action='store_true')
-
-args = [a for a in sys.argv if a != '-h']
-args, _ = parser.parse_known_args(args)
-
-# Configure initial bootstrap
-rb = RustBuild()
-rb.config_toml = ''
-rb.config_mk = ''
-rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
-rb.build_dir = os.path.join(os.getcwd(), "build")
-rb.verbose = args.verbose
-
-try:
-    with open(args.config or 'config.toml') as config:
-        rb.config_toml = config.read()
-except:
-    pass
-try:
-    rb.config_mk = open('config.mk').read()
-except:
-    pass
-
-# Fetch/build the bootstrap
-rb.build = rb.build_triple()
-rb.parse_nightly_dates()
-rb.download_rust_nightly()
-sys.stdout.flush()
-rb.build_bootstrap()
-sys.stdout.flush()
-
-# Run the bootstrap
-args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
-args.append('--src')
-args.append(rb.rust_root)
-args.append('--build')
-args.append(rb.build)
-args.extend(sys.argv[1:])
-env = os.environ.copy()
-env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
-rb.run(args, env)
+def main():
+    parser = argparse.ArgumentParser(description='Build rust')
+    parser.add_argument('--config')
+    parser.add_argument('-v', '--verbose', action='store_true')
+
+    args = [a for a in sys.argv if a != '-h']
+    args, _ = parser.parse_known_args(args)
+
+    # Configure initial bootstrap
+    rb = RustBuild()
+    rb.config_toml = ''
+    rb.config_mk = ''
+    rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
+    rb.build_dir = os.path.join(os.getcwd(), "build")
+    rb.verbose = args.verbose
+
+    try:
+        with open(args.config or 'config.toml') as config:
+            rb.config_toml = config.read()
+    except:
+        pass
+    try:
+        rb.config_mk = open('config.mk').read()
+    except:
+        pass
+
+    data = stage0_data(rb.rust_root)
+    rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
+    rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)
+
+    # Fetch/build the bootstrap
+    rb.build = rb.build_triple()
+    rb.download_stage0()
+    sys.stdout.flush()
+    rb.build_bootstrap()
+    sys.stdout.flush()
+
+    # Run the bootstrap
+    args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
+    args.append('--src')
+    args.append(rb.rust_root)
+    args.append('--build')
+    args.append(rb.build)
+    args.extend(sys.argv[1:])
+    env = os.environ.copy()
+    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
+    rb.run(args, env)
+
+if __name__ == '__main__':
+    main()

src/bootstrap/build/channel.rs

@@ -84,4 +84,12 @@ pub fn collect(build: &mut Build) {
     build.bootstrap_key = format!("{:02x}{:02x}{:02x}{:02x}",
                                   key[0], key[1], key[2], key[3]);
     env::set_var("RUSTC_BOOTSTRAP_KEY", &build.bootstrap_key);
+
+    let mut s = String::new();
+    t!(t!(File::open(build.src.join("src/stage0.txt"))).read_to_string(&mut s));
+    if let Some(line) = s.lines().find(|l| l.starts_with("rustc_key")) {
+        if let Some(key) = line.split(": ").nth(1) {
+            build.bootstrap_key_stage0 = key.to_string();
+        }
+    }
 }

src/bootstrap/build/compile.rs

@@ -179,7 +179,6 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
          .env("CFG_VERSION", &build.version)
          .env("CFG_BOOTSTRAP_KEY", &build.bootstrap_key)
          .env("CFG_PREFIX", build.config.prefix.clone().unwrap_or(String::new()))
-         .env("RUSTC_BOOTSTRAP_KEY", &build.bootstrap_key)
          .env("CFG_LIBDIR_RELATIVE", "lib");
 
     if let Some(ref ver_date) = build.ver_date {