From 11f9283da91ca4ab7fb634f3f586156f699007c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 12 Sep 2023 19:33:09 +0200 Subject: [PATCH] Add a Local environment to `opt-dist` This makes it easier to build a PGO/BOLT optimized `rustc` locally, outside of CI. --- src/tools/opt-dist/src/environment.rs | 7 +-- src/tools/opt-dist/src/main.rs | 75 ++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/src/tools/opt-dist/src/environment.rs b/src/tools/opt-dist/src/environment.rs index e730e61060431..ed43f469c5e91 100644 --- a/src/tools/opt-dist/src/environment.rs +++ b/src/tools/opt-dist/src/environment.rs @@ -7,7 +7,8 @@ pub struct Environment { python_binary: String, /// The rustc checkout, where the compiler source is located. checkout_dir: Utf8PathBuf, - /// The main directory where the build occurs. + /// The main directory where the build occurs. Stage0 rustc and cargo have to be available in + /// this directory before `opt-dist` is started. build_dir: Utf8PathBuf, /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.) /// will be stored. @@ -15,7 +16,7 @@ pub struct Environment { /// Path to the host LLVM used to compile LLVM in `src/llvm-project`. host_llvm_dir: Utf8PathBuf, /// List of test paths that should be skipped when testing the optimized artifacts. - skipped_tests: Vec<&'static str>, + skipped_tests: Vec, use_bolt: bool, shared_llvm: bool, } @@ -83,7 +84,7 @@ impl Environment { self.shared_llvm } - pub fn skipped_tests(&self) -> &[&'static str] { + pub fn skipped_tests(&self) -> &[String] { &self.skipped_tests } } diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index 6a3bdd7cc75b4..b5e123360f875 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -42,10 +42,51 @@ struct SharedArgs { #[derive(clap::Parser, Clone, Debug)] enum EnvironmentCmd { + /// Perform a custom local PGO/BOLT optimized build. + Local { + /// Target triple of the host. + #[arg(long)] + target_triple: String, + + /// Checkout directory of `rustc`. + #[arg(long)] + checkout_dir: Utf8PathBuf, + + /// Host LLVM installation directory. + #[arg(long)] + llvm_dir: Utf8PathBuf, + + /// Python binary to use in bootstrap invocations. + #[arg(long, default_value = "python3")] + python: String, + + /// Directory where artifacts (like PGO profiles or rustc-perf) of this workflow + /// will be stored. + #[arg(long, default_value = "opt-artifacts")] + artifact_dir: Utf8PathBuf, + + /// Is LLVM for `rustc` built in shared library mode? + #[arg(long, default_value_t = true)] + llvm_shared: bool, + + /// Should BOLT optimization be used? If yes, host LLVM must have BOLT binaries + /// (`llvm-bolt` and `merge-fdata`) available. + #[arg(long, default_value_t = false)] + use_bolt: bool, + + /// Tests that should be skipped when testing the optimized compiler. + #[arg(long)] + skipped_tests: Vec, + + #[clap(flatten)] + shared: SharedArgs, + }, + /// Perform an optimized build on Linux CI, from inside Docker. LinuxCi { #[clap(flatten)] shared: SharedArgs, }, + /// Perform an optimized build on Windows CI, directly inside Github Actions. WindowsCi { #[clap(flatten)] shared: SharedArgs, @@ -58,6 +99,34 @@ fn is_try_build() -> bool { fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> { let (env, args) = match args.env { + EnvironmentCmd::Local { + target_triple, + checkout_dir, + llvm_dir, + python, + artifact_dir, + llvm_shared, + use_bolt, + skipped_tests, + shared, + } => { + let env = EnvironmentBuilder::default() + .host_triple(target_triple) + .python_binary(python) + .checkout_dir(checkout_dir.clone()) + .host_llvm_dir(llvm_dir) + .artifact_dir(artifact_dir) + .build_dir(checkout_dir) + .shared_llvm(llvm_shared) + .use_bolt(use_bolt) + .skipped_tests(skipped_tests) + .build()?; + with_log_group("Building rustc-perf", || { + Ok::<(), anyhow::Error>(download_rustc_perf(&env)?) + })?; + + (env, shared.build_args) + } EnvironmentCmd::LinuxCi { shared } => { let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); @@ -74,7 +143,7 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .use_bolt(true) .skipped_tests(vec![ // Fails because of linker errors, as of June 2023. - "tests/ui/process/nofile-limit.rs", + "tests/ui/process/nofile-limit.rs".to_string(), ]) .build()?; // /tmp/rustc-perf comes from the x64 dist Dockerfile @@ -100,7 +169,7 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .use_bolt(false) .skipped_tests(vec![ // Fails as of June 2023. - "tests\\codegen\\vec-shrink-panik.rs", + "tests\\codegen\\vec-shrink-panik.rs".to_string(), ]) .build()?; @@ -309,6 +378,8 @@ fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> { // Download and build rustc-perf into the given environment. fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> { + reset_directory(&env.rustc_perf_dir())?; + // FIXME: add some mechanism for synchronization of this commit SHA with // Linux (which builds rustc-perf in a Dockerfile) // rustc-perf version from 2023-05-30