diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml index 8e6c1e8ade015..652d6eca3f6e9 100644 --- a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml +++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml @@ -12,9 +12,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install rustfmt + - name: Avoid installing rustc-dev run: | - rustup component add rustfmt + sed -i 's/components.*/components = ["rustfmt"]/' rust-toolchain + echo 'profile = "minimal"' >> rust-toolchain + rustfmt -v - name: Rustfmt run: | @@ -127,7 +129,7 @@ jobs: - uses: actions/checkout@v3 - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Disable JIT tests run: | @@ -136,7 +138,7 @@ jobs: - name: Test env: TARGET_TRIPLE: x86_64-unknown-linux-gnu - run: ./y.rs test --use-backend llvm + run: ./y.sh test --use-backend llvm bench: runs-on: ubuntu-latest diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock index 904233d424276..af8e43da4eafe 100644 --- a/compiler/rustc_codegen_cranelift/Cargo.lock +++ b/compiler/rustc_codegen_cranelift/Cargo.lock @@ -51,18 +51,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6160c0a96253993b79fb7e0983534a4515ecf666120ddf8f92068114997ebc" +checksum = "ec27af72e56235eb326b5bf2de4e70ab7c5ac1fb683a1829595badaf821607fd" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b38da5f63562e42f3c929d7c76871098e5ad12c8ab44b0659ffc529f22a5b3a" +checksum = "2231e12925e6c5f4bc9c95b62a798eea6ed669a95bc3e00f8b2adb3b7b9b7a80" dependencies = [ "bumpalo", "cranelift-bforest", @@ -81,39 +81,39 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "011371e213e163b55dd9e8404b3f2d9fa52cd14dc2f3dc5b83e61ffceff126db" +checksum = "413b00b8dfb3aab85674a534677e7ca08854b503f164a70ec0634fce80996e2c" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf97dde7f5ad571161cdd203a2c9c88682ef669830aea3c14ea5d164ef8bb43" +checksum = "cd0feb9ecc8193ef5cb04f494c5bd835e5bfec4bde726e7ac0444fc9dd76229e" [[package]] name = "cranelift-control" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd9a9254aee733b0f2b68e5eaaf0337ad53cb23252a056c10a35370551be8d40" +checksum = "72eedd2afcf5fee1e042eaaf18d3750e48ad0eca364a9f5971ecfdd5ef85bf71" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf39a33ee39479d1337cd9333f3c09786c5a0ca1ec509edcaf9d1346d5de0e5" +checksum = "7af19157be42671073cf8c2a52d6a4ae1e7b11f1dcb4131fede356d9f91c29dd" [[package]] name = "cranelift-frontend" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e260b92a193a0a2dccc3938f133d9532e7dcfe8d03e36bf8b7d3518c1c1793" +checksum = "c2dc7636c5fad156be7d9ae691cd1aaecd97326caf2ab534ba168056d56aa76c" dependencies = [ "cranelift-codegen", "log", @@ -123,15 +123,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9446c8e1aadfcdacee1a49592bc2c25d1d9bf5484782c163e7f5485c92cd3c1c" +checksum = "c1111aea4fb6fade5779903f184249a3fc685a799fe4ec59126f9af59c7c2a74" [[package]] name = "cranelift-jit" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "689a6df165d0f860c1e1a3d53c28944e2743c3e9ee4c678cf190fe60ad7a6ef5" +checksum = "dadf88076317f6286ec77ebbe65978734fb43b6befdc96f52ff4c4c511841644" dependencies = [ "anyhow", "cranelift-codegen", @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b1402d6ff1695b429536b2eaa126db560fc94c375ed0e9cfb15051fc07427f7" +checksum = "c6bae8a82dbf82241b1083e57e06870d2c2bdc9852727be99d58477513816953" dependencies = [ "anyhow", "cranelift-codegen", @@ -160,9 +160,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac916f3c5aff4b817e42fc2e682292b931495b3fe2603d5e3c3cf602d74e344" +checksum = "1ecfc01a634448468a698beac433d98040033046678a0eed3ca39a3a9f63ae86" dependencies = [ "cranelift-codegen", "libc", @@ -171,9 +171,9 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.96.1" +version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23860f4cd064017f2108e6bc5d25660a77cd6eea77f1ac0756870a00abb12e93" +checksum = "0ee14a7276999f0dcaae2de84043e2c2de50820fb89b3db56fab586a4ad26734" dependencies = [ "anyhow", "cranelift-codegen", @@ -193,6 +193,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -206,7 +212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" dependencies = [ "fallible-iterator", - "indexmap", + "indexmap 1.9.3", "stable_deref_trait", ] @@ -225,6 +231,12 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "indexmap" version = "1.9.3" @@ -235,6 +247,16 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "libc" version = "0.2.138" @@ -283,7 +305,7 @@ checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" dependencies = [ "crc32fast", "hashbrown 0.13.2", - "indexmap", + "indexmap 1.9.3", "memchr", ] @@ -295,9 +317,9 @@ checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "regalloc2" -version = "0.8.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a52e724646c6c0800fc456ec43b4165d2f91fba88ceaca06d9e0b400023478" +checksum = "5b4dcbd3a2ae7fb94b5813fa0e957c6ab51bf5d0a8ee1b69e0c2d0f1e6eb8485" dependencies = [ "hashbrown 0.13.2", "log", @@ -335,7 +357,7 @@ dependencies = [ "cranelift-native", "cranelift-object", "gimli", - "indexmap", + "indexmap 2.0.0", "libloading", "object", "smallvec", @@ -374,9 +396,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "wasmtime-jit-icache-coherence" -version = "9.0.1" +version = "11.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d90933b781e1cef7656baed671c7a90bdba0c1c694e04fdd4124419308f5cbb" +checksum = "e34eb67f0829a5614ec54716c8e0c9fe68fab7b9df3686c85f719c9d247f7169" dependencies = [ "cfg-if", "libc", diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml index 1c1f2d8577bd6..8ded81d7399b5 100644 --- a/compiler/rustc_codegen_cranelift/Cargo.toml +++ b/compiler/rustc_codegen_cranelift/Cargo.toml @@ -8,17 +8,17 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { version = "0.96.1", features = ["unwind", "all-arch"] } -cranelift-frontend = { version = "0.96.1" } -cranelift-module = { version = "0.96.1" } -cranelift-native = { version = "0.96.1" } -cranelift-jit = { version = "0.96.1", optional = true } -cranelift-object = { version = "0.96.1" } +cranelift-codegen = { version = "0.98", features = ["unwind", "all-arch"] } +cranelift-frontend = { version = "0.98" } +cranelift-module = { version = "0.98" } +cranelift-native = { version = "0.98" } +cranelift-jit = { version = "0.98", optional = true } +cranelift-object = { version = "0.98" } target-lexicon = "0.12.0" gimli = { version = "0.27.2", default-features = false, features = ["write"]} object = { version = "0.30.3", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } -indexmap = "1.9.3" +indexmap = "2.0.0" libloading = { version = "0.7.3", optional = true } smallvec = "1.8.1" diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md index 9469feea0cbf8..62eaef359af92 100644 --- a/compiler/rustc_codegen_cranelift/Readme.md +++ b/compiler/rustc_codegen_cranelift/Readme.md @@ -65,12 +65,12 @@ to `./build/host/stage2/bin/`. Note that you would need to do this every time yo 5. Copy cargo from another toolchain: `cp $(rustup which cargo) .build//stage2/bin/cargo` * Another option is to build it at step 3 and copy with other executables at step 4. 6. Link your new `rustc` to toolchain: `rustup toolchain link stage2 ./build/host/stage2/`. -7. (Windows only) compile y.rs: `rustc +stage2 -O y.rs`. -8. You need to prefix every `./y.rs` (or `y` if you built `y.rs`) command by `rustup run stage2` to make cg_clif use your local changes in rustc. +7. (Windows only) compile the build system: `rustc +stage2 -O build_system/main.rs -o y.exe`. +8. You need to prefix every `./y.sh` (or `y` if you built `build_system/main.rs` as `y`) command by `rustup run stage2` to make cg_clif use your local changes in rustc. - * `rustup run stage2 ./y.rs prepare` - * `rustup run stage2 ./y.rs build` - * (Optional) run tests: `rustup run stage2 ./y.rs test` + * `rustup run stage2 ./y.sh prepare` + * `rustup run stage2 ./y.sh build` + * (Optional) run tests: `rustup run stage2 ./y.sh test` 9. Now you can use your cg_clif build to compile other Rust programs, e.g. you can open any Rust crate and run commands like `$RustCheckoutDir/compiler/rustc_codegen_cranelift/dist/cargo-clif build --release`. ## Configuration diff --git a/compiler/rustc_codegen_cranelift/build_system/bench.rs b/compiler/rustc_codegen_cranelift/build_system/bench.rs index 2bb1180003448..cec608ea0422d 100644 --- a/compiler/rustc_codegen_cranelift/build_system/bench.rs +++ b/compiler/rustc_codegen_cranelift/build_system/bench.rs @@ -1,4 +1,5 @@ use std::env; +use std::io::Write; use std::path::Path; use super::path::{Dirs, RelPath}; @@ -30,6 +31,12 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) { let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap(); + let mut gha_step_summary = if let Ok(file) = std::env::var("GITHUB_STEP_SUMMARY") { + Some(std::fs::OpenOptions::new().append(true).open(file).unwrap()) + } else { + None + }; + eprintln!("[BENCH COMPILE] ebobby/simple-raytracer"); let cargo_clif = RelPath::DIST .to_path(dirs) @@ -60,36 +67,64 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) { target_dir = target_dir.display(), ); + let bench_compile_markdown = RelPath::DIST.to_path(dirs).join("bench_compile.md"); + let bench_compile = hyperfine_command( 1, bench_runs, Some(&clean_cmd), - &[&llvm_build_cmd, &clif_build_cmd, &clif_build_opt_cmd], + &[ + ("cargo build", &llvm_build_cmd), + ("cargo-clif build", &clif_build_cmd), + ("cargo-clif build --release", &clif_build_opt_cmd), + ], + &bench_compile_markdown, ); spawn_and_wait(bench_compile); + if let Some(gha_step_summary) = gha_step_summary.as_mut() { + gha_step_summary.write_all(b"## Compile ebobby/simple-raytracer\n\n").unwrap(); + gha_step_summary.write_all(&std::fs::read(bench_compile_markdown).unwrap()).unwrap(); + gha_step_summary.write_all(b"\n").unwrap(); + } + eprintln!("[BENCH RUN] ebobby/simple-raytracer"); + let bench_run_markdown = RelPath::DIST.to_path(dirs).join("bench_run.md"); + + let raytracer_cg_llvm = Path::new(".").join(get_file_name( + &bootstrap_host_compiler.rustc, + "raytracer_cg_llvm", + "bin", + )); + let raytracer_cg_clif = Path::new(".").join(get_file_name( + &bootstrap_host_compiler.rustc, + "raytracer_cg_clif", + "bin", + )); + let raytracer_cg_clif_opt = Path::new(".").join(get_file_name( + &bootstrap_host_compiler.rustc, + "raytracer_cg_clif_opt", + "bin", + )); let mut bench_run = hyperfine_command( 0, bench_runs, None, &[ - Path::new(".") - .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_llvm", "bin")) - .to_str() - .unwrap(), - Path::new(".") - .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif", "bin")) - .to_str() - .unwrap(), - Path::new(".") - .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif_opt", "bin")) - .to_str() - .unwrap(), + ("", raytracer_cg_llvm.to_str().unwrap()), + ("", raytracer_cg_clif.to_str().unwrap()), + ("", raytracer_cg_clif_opt.to_str().unwrap()), ], + &bench_run_markdown, ); bench_run.current_dir(RelPath::BUILD.to_path(dirs)); spawn_and_wait(bench_run); + + if let Some(gha_step_summary) = gha_step_summary.as_mut() { + gha_step_summary.write_all(b"## Run ebobby/simple-raytracer\n\n").unwrap(); + gha_step_summary.write_all(&std::fs::read(bench_run_markdown).unwrap()).unwrap(); + gha_step_summary.write_all(b"\n").unwrap(); + } } diff --git a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs index 6855c1a7fc5bf..1c5db23299d6a 100644 --- a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs +++ b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use super::path::{Dirs, RelPath}; use super::rustc_info::get_file_name; -use super::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler}; +use super::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler, LogGroup}; pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif"); @@ -13,6 +13,8 @@ pub(crate) fn build_backend( bootstrap_host_compiler: &Compiler, use_unstable_features: bool, ) -> PathBuf { + let _group = LogGroup::guard("Build backend"); + let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs); maybe_incremental(&mut cmd); diff --git a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs index 74bba9ed5eb88..04097936d03d0 100644 --- a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs +++ b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs @@ -6,6 +6,7 @@ use super::path::{Dirs, RelPath}; use super::rustc_info::get_file_name; use super::utils::{ maybe_incremental, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, + LogGroup, }; use super::{CodegenBackend, SysrootKind}; @@ -22,6 +23,8 @@ pub(crate) fn build_sysroot( rustup_toolchain_name: Option<&str>, target_triple: String, ) -> Compiler { + let _guard = LogGroup::guard("Build sysroot"); + eprintln!("[BUILD] sysroot {:?}", sysroot_kind); DIST_DIR.ensure_fresh(dirs); @@ -251,7 +254,10 @@ fn build_clif_sysroot_for_triple( rustflags .push_str(&format!(" --sysroot {}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap())); if channel == "release" { - rustflags.push_str(" -Zmir-opt-level=3"); + // Incremental compilation by default disables mir inlining. This leads to both a decent + // compile perf and a significant runtime perf regression. As such forcefully enable mir + // inlining. + rustflags.push_str(" -Zinline-mir"); } compiler.rustflags += &rustflags; let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs); diff --git a/compiler/rustc_codegen_cranelift/build_system/prepare.rs b/compiler/rustc_codegen_cranelift/build_system/prepare.rs index e31e39a483fe3..3ee2e8f4a4e34 100644 --- a/compiler/rustc_codegen_cranelift/build_system/prepare.rs +++ b/compiler/rustc_codegen_cranelift/build_system/prepare.rs @@ -27,6 +27,7 @@ pub(crate) fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { STDLIB_SRC.to_path(dirs).join("Cargo.toml"), r#" [workspace] +resolver = "1" members = ["./library/sysroot"] [patch.crates-io] diff --git a/compiler/rustc_codegen_cranelift/build_system/tests.rs b/compiler/rustc_codegen_cranelift/build_system/tests.rs index 08d8f708c7da4..0254d18cf7c2d 100644 --- a/compiler/rustc_codegen_cranelift/build_system/tests.rs +++ b/compiler/rustc_codegen_cranelift/build_system/tests.rs @@ -3,7 +3,7 @@ use super::config; use super::path::{Dirs, RelPath}; use super::prepare::{apply_patches, GitRepo}; use super::rustc_info::get_default_sysroot; -use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler}; +use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler, LogGroup}; use super::{CodegenBackend, SysrootKind}; use std::env; use std::ffi::OsStr; @@ -21,6 +21,7 @@ struct TestCase { enum TestCaseCmd { Custom { func: &'static dyn Fn(&TestRunner<'_>) }, BuildLib { source: &'static str, crate_types: &'static str }, + BuildBin { source: &'static str }, BuildBinAndRun { source: &'static str, args: &'static [&'static str] }, JitBin { source: &'static str, args: &'static str }, } @@ -39,6 +40,10 @@ impl TestCase { Self { config, cmd: TestCaseCmd::BuildLib { source, crate_types } } } + const fn build_bin(config: &'static str, source: &'static str) -> Self { + Self { config, cmd: TestCaseCmd::BuildBin { source } } + } + const fn build_bin_and_run( config: &'static str, source: &'static str, @@ -92,6 +97,7 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[ TestCase::build_bin_and_run("aot.float-minmax-pass", "example/float-minmax-pass.rs", &[]), TestCase::build_bin_and_run("aot.mod_bench", "example/mod_bench.rs", &[]), TestCase::build_bin_and_run("aot.issue-72793", "example/issue-72793.rs", &[]), + TestCase::build_bin("aot.issue-59326", "example/issue-59326.rs"), ]; // FIXME(rust-random/rand#1293): Newer rand versions fail to test on Windows. Update once this is @@ -119,8 +125,8 @@ pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( "rust-lang", "portable-simd", - "ad8afa8c81273b3b49acbea38cd3bcf17a34cf2b", - "800548f8000e31bd", + "7c7dbe0c505ccbc02ff30c1e37381ab1d47bf46f", + "5bcc9c544f6fa7bd", "portable-simd", ); @@ -380,15 +386,17 @@ impl<'a> TestRunner<'a> { let tag = tag.to_uppercase(); let is_jit_test = tag == "JIT"; - if !config::get_bool(config) + let _guard = if !config::get_bool(config) || (is_jit_test && !self.jit_supported) || self.skip_tests.contains(&config) { eprintln!("[{tag}] {testname} (skipped)"); continue; } else { + let guard = LogGroup::guard(&format!("[{tag}] {testname}")); eprintln!("[{tag}] {testname}"); - } + guard + }; match *cmd { TestCaseCmd::Custom { func } => func(self), @@ -405,6 +413,13 @@ impl<'a> TestRunner<'a> { ]); } } + TestCaseCmd::BuildBin { source } => { + if self.use_unstable_features { + self.run_rustc([source]); + } else { + self.run_rustc([source, "--cfg", "no_unstable_features"]); + } + } TestCaseCmd::BuildBinAndRun { source, args } => { if self.use_unstable_features { self.run_rustc([source]); diff --git a/compiler/rustc_codegen_cranelift/build_system/utils.rs b/compiler/rustc_codegen_cranelift/build_system/utils.rs index 41fc366e2903c..97c82d501c5a9 100644 --- a/compiler/rustc_codegen_cranelift/build_system/utils.rs +++ b/compiler/rustc_codegen_cranelift/build_system/utils.rs @@ -3,6 +3,7 @@ use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process::{self, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; use super::path::{Dirs, RelPath}; @@ -136,10 +137,13 @@ pub(crate) fn hyperfine_command( warmup: u64, runs: u64, prepare: Option<&str>, - cmds: &[&str], + cmds: &[(&str, &str)], + markdown_export: &Path, ) -> Command { let mut bench = Command::new("hyperfine"); + bench.arg("--export-markdown").arg(markdown_export); + if warmup != 0 { bench.arg("--warmup").arg(warmup.to_string()); } @@ -152,7 +156,12 @@ pub(crate) fn hyperfine_command( bench.arg("--prepare").arg(prepare); } - bench.args(cmds); + for &(name, cmd) in cmds { + if name != "" { + bench.arg("-n").arg(name); + } + bench.arg(cmd); + } bench } @@ -167,6 +176,8 @@ pub(crate) fn git_command<'a>(repo_dir: impl Into>, cmd: &str) .arg("user.email=dummy@example.com") .arg("-c") .arg("core.autocrlf=false") + .arg("-c") + .arg("commit.gpgSign=false") .arg(cmd); if let Some(repo_dir) = repo_dir.into() { git_cmd.current_dir(repo_dir); @@ -259,6 +270,33 @@ pub(crate) fn is_ci_opt() -> bool { env::var("CI_OPT").is_ok() } +static IN_GROUP: AtomicBool = AtomicBool::new(false); +pub(crate) struct LogGroup { + is_gha: bool, +} + +impl LogGroup { + pub(crate) fn guard(name: &str) -> LogGroup { + let is_gha = env::var("GITHUB_ACTIONS").is_ok(); + + assert!(!IN_GROUP.swap(true, Ordering::SeqCst)); + if is_gha { + eprintln!("::group::{name}"); + } + + LogGroup { is_gha } + } +} + +impl Drop for LogGroup { + fn drop(&mut self) { + if self.is_gha { + eprintln!("::endgroup::"); + } + IN_GROUP.store(false, Ordering::SeqCst); + } +} + pub(crate) fn maybe_incremental(cmd: &mut Command) { if is_ci() || std::env::var("CARGO_BUILD_INCREMENTAL").map_or(false, |val| val == "false") { // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway diff --git a/compiler/rustc_codegen_cranelift/config.txt b/compiler/rustc_codegen_cranelift/config.txt index d6e3924a24d64..fa1c9f4259c49 100644 --- a/compiler/rustc_codegen_cranelift/config.txt +++ b/compiler/rustc_codegen_cranelift/config.txt @@ -41,6 +41,7 @@ aot.track-caller-attribute aot.float-minmax-pass aot.mod_bench aot.issue-72793 +aot.issue-59326 testsuite.extended_sysroot test.rust-random/rand diff --git a/compiler/rustc_codegen_cranelift/example/issue-59326.rs b/compiler/rustc_codegen_cranelift/example/issue-59326.rs new file mode 100644 index 0000000000000..70b7c94e15c11 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/issue-59326.rs @@ -0,0 +1,27 @@ +// Based on https://github.com/rust-lang/rust/blob/689511047a75a30825e367d4fd45c74604d0b15e/tests/ui/issues/issue-59326.rs#L1 +// check-pass +trait Service { + type S; +} + +trait Framing { + type F; +} + +impl Framing for () { + type F = (); +} + +trait HttpService: Service {} + +type BoxService = Box>; + +fn build_server BoxService>(_: F) {} + +fn make_server() -> Box> { + unimplemented!() +} + +fn main() { + build_server(|| make_server()) +} diff --git a/compiler/rustc_codegen_cranelift/example/mini_core.rs b/compiler/rustc_codegen_cranelift/example/mini_core.rs index 79ca4c039854e..9ecc4c5dd5b64 100644 --- a/compiler/rustc_codegen_cranelift/example/mini_core.rs +++ b/compiler/rustc_codegen_cranelift/example/mini_core.rs @@ -547,7 +547,9 @@ impl Box { impl Drop for Box { fn drop(&mut self) { // inner value is dropped by compiler - libc::free(self.0.pointer.0 as *mut u8); + unsafe { + libc::free(self.0.pointer.0 as *mut u8); + } } } diff --git a/compiler/rustc_codegen_cranelift/example/std_example.rs b/compiler/rustc_codegen_cranelift/example/std_example.rs index 1bf0ff64c9249..490cc2404f626 100644 --- a/compiler/rustc_codegen_cranelift/example/std_example.rs +++ b/compiler/rustc_codegen_cranelift/example/std_example.rs @@ -1,4 +1,12 @@ -#![feature(core_intrinsics, generators, generator_trait, is_sorted, repr_simd)] +#![feature( + core_intrinsics, + generators, + generator_trait, + is_sorted, + repr_simd, + tuple_trait, + unboxed_closures +)] #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; @@ -155,12 +163,34 @@ fn main() { } foo(I64X2(0, 0)); + + transmute_fat_pointer(); + + rust_call_abi(); } fn panic(_: u128) { panic!(); } +use std::mem::transmute; + +#[cfg(target_pointer_width = "32")] +type TwoPtrs = i64; +#[cfg(target_pointer_width = "64")] +type TwoPtrs = i128; + +fn transmute_fat_pointer() -> TwoPtrs { + unsafe { transmute::<_, TwoPtrs>("true !") } +} + +extern "rust-call" fn rust_call_abi_callee(_: T) {} + +fn rust_call_abi() { + rust_call_abi_callee(()); + rust_call_abi_callee((1, 2)); +} + #[repr(simd)] struct I64X2(i64, i64); diff --git a/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch index 1d5479beddee9..a650e10110bf2 100644 --- a/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch +++ b/compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch @@ -10,6 +10,18 @@ Cranelift doesn't support them yet library/core/tests/atomic.rs | 4 --- 4 files changed, 4 insertions(+), 50 deletions(-) +diff --git a/lib.rs b/lib.rs +index 897a5e9..331f66f 100644 +--- a/lib.rs ++++ b/lib.rs +@@ -93,7 +93,6 @@ + #![feature(const_option)] + #![feature(const_option_ext)] + #![feature(const_result)] +-#![cfg_attr(target_has_atomic = "128", feature(integer_atomics))] + #![feature(int_roundings)] + #![feature(slice_group_by)] + #![feature(split_array)] diff --git a/atomic.rs b/atomic.rs index b735957..ea728b6 100644 --- a/atomic.rs diff --git a/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch index 45f73f36b9317..646928893e96a 100644 --- a/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch +++ b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch @@ -38,9 +38,9 @@ diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index d9de37e..8293fce 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs -@@ -2234,46 +2234,6 @@ atomic_int! { - "AtomicU64::new(0)", - u64 AtomicU64 ATOMIC_U64_INIT +@@ -2996,42 +2996,6 @@ atomic_int! { + 8, + u64 AtomicU64 } -#[cfg(target_has_atomic_load_store = "128")] -atomic_int! { @@ -53,14 +53,12 @@ index d9de37e..8293fce 100644 - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), -- unstable(feature = "integer_atomics", issue = "99069"), - cfg_attr(not(test), rustc_diagnostic_item = "AtomicI128"), - "i128", - "#![feature(integer_atomics)]\n\n", - atomic_min, atomic_max, - 16, -- "AtomicI128::new(0)", -- i128 AtomicI128 ATOMIC_I128_INIT +- i128 AtomicI128 -} -#[cfg(target_has_atomic_load_store = "128")] -atomic_int! { @@ -73,16 +71,15 @@ index d9de37e..8293fce 100644 - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"), -- unstable(feature = "integer_atomics", issue = "99069"), - cfg_attr(not(test), rustc_diagnostic_item = "AtomicU128"), - "u128", - "#![feature(integer_atomics)]\n\n", - atomic_umin, atomic_umax, - 16, -- "AtomicU128::new(0)", -- u128 AtomicU128 ATOMIC_U128_INIT +- u128 AtomicU128 -} + #[cfg(target_has_atomic_load_store = "ptr")] macro_rules! atomic_int_ptr_sized { ( $($target_pointer_width:literal $align:literal)* ) => { $( -- diff --git a/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml b/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml index 1dde9e54d7ee0..aea47bdfba262 100644 --- a/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml +++ b/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" dependencies = [ "compiler_builtins", "gimli", @@ -34,6 +34,12 @@ dependencies = [ "rand_xorshift", ] +[[package]] +name = "allocator-api2" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" + [[package]] name = "auxv" version = "0.3.3" @@ -68,9 +74,9 @@ dependencies = [ [[package]] name = "compiler_builtins" -version = "0.1.93" +version = "0.1.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76630810d973ecea3dbf611e1b7aecfb1012751ef1ff8de3998f89014a166781" +checksum = "6866e0f3638013234db3c89ead7a14d278354338e7237257407500009012b23f" dependencies = [ "cc", "rustc-std-workspace-core", @@ -145,10 +151,11 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" dependencies = [ + "allocator-api2", "compiler_builtins", "rustc-std-workspace-alloc", "rustc-std-workspace-core", @@ -156,9 +163,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" dependencies = [ "compiler_builtins", "rustc-std-workspace-alloc", @@ -186,9 +193,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", "compiler_builtins", @@ -198,9 +205,9 @@ dependencies = [ [[package]] name = "object" -version = "0.30.4" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" dependencies = [ "compiler_builtins", "memchr", diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain index fa3a10b9adc6f..34514658359cb 100644 --- a/compiler/rustc_codegen_cranelift/rust-toolchain +++ b/compiler/rustc_codegen_cranelift/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2023-06-15" +channel = "nightly-2023-07-22" components = ["rust-src", "rustc-dev", "llvm-tools"] diff --git a/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs b/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs index 99b97be24e69b..f73b2012684cd 100644 --- a/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs +++ b/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs @@ -40,14 +40,22 @@ fn main() { "cargo" }; - let args: Vec<_> = match env::args().nth(1).as_deref() { + let mut args = env::args().skip(1).collect::>(); + if args.get(0).map(|arg| &**arg) == Some("clif") { + // Avoid infinite recursion when invoking `cargo-clif` as cargo subcommand using + // `cargo clif`. + args.remove(0); + } + + let args: Vec<_> = match args.get(0).map(|arg| &**arg) { Some("jit") => { env::set_var( "RUSTFLAGS", env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic", ); + args.remove(0); IntoIterator::into_iter(["rustc".to_string()]) - .chain(env::args().skip(2)) + .chain(args) .chain([ "--".to_string(), "-Zunstable-options".to_string(), @@ -60,8 +68,9 @@ fn main() { "RUSTFLAGS", env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic", ); + args.remove(0); IntoIterator::into_iter(["rustc".to_string()]) - .chain(env::args().skip(2)) + .chain(args) .chain([ "--".to_string(), "-Zunstable-options".to_string(), @@ -69,7 +78,7 @@ fn main() { ]) .collect() } - _ => env::args().skip(1).collect(), + _ => args, }; #[cfg(unix)] diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh index 15b16b42be5b9..e6bbac647e5a2 100644 --- a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh +++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh @@ -10,7 +10,8 @@ git fetch git checkout -- . git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')" -git -c user.name=Dummy -c user.email=dummy@example.com am ../patches/*-stdlib-*.patch +git -c user.name=Dummy -c user.email=dummy@example.com -c commit.gpgSign=false \ + am ../patches/*-stdlib-*.patch git apply - <( // Unpack arguments tuple for closures let mut args = if fn_sig.abi() == Abi::RustCall { - assert_eq!(args.len(), 2, "rust-call abi requires two arguments"); - let self_arg = codegen_call_argument_operand(fx, &args[0]); - let pack_arg = codegen_call_argument_operand(fx, &args[1]); + let (self_arg, pack_arg) = match args { + [pack_arg] => (None, codegen_call_argument_operand(fx, pack_arg)), + [self_arg, pack_arg] => ( + Some(codegen_call_argument_operand(fx, self_arg)), + codegen_call_argument_operand(fx, pack_arg), + ), + _ => panic!("rust-call abi requires one or two arguments"), + }; let tupled_arguments = match pack_arg.value.layout().ty.kind() { ty::Tuple(ref tupled_arguments) => tupled_arguments, @@ -455,7 +460,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( }; let mut args = Vec::with_capacity(1 + tupled_arguments.len()); - args.push(self_arg); + args.extend(self_arg); for i in 0..tupled_arguments.len() { args.push(CallArgument { value: pack_arg.value.value_field(fx, FieldIdx::new(i)), diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index e05f2146f0c30..522dd7189fe63 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -7,6 +7,8 @@ use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; use cranelift_codegen::ir::UserFuncName; +use cranelift_codegen::CodegenError; +use cranelift_module::ModuleError; use crate::constant::ConstantCx; use crate::debuginfo::FunctionDebugContext; @@ -172,7 +174,21 @@ pub(crate) fn compile_fn( // Define function cx.profiler.generic_activity("define function").run(|| { context.want_disasm = cx.should_write_ir; - module.define_function(codegened_func.func_id, context).unwrap(); + match module.define_function(codegened_func.func_id, context) { + Ok(()) => {} + Err(ModuleError::Compilation(CodegenError::ImplLimitExceeded)) => { + let handler = rustc_session::EarlyErrorHandler::new( + rustc_session::config::ErrorOutputType::default(), + ); + handler.early_error(format!( + "backend implementation limit exceeded while compiling {name}", + name = codegened_func.symbol_name + )); + } + Err(err) => { + panic!("Error while defining {name}: {err:?}", name = codegened_func.symbol_name); + } + } }); if cx.should_write_ir { @@ -356,7 +372,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { codegen_panic_inner( fx, - rustc_hir::LangItem::PanicBoundsCheck, + rustc_hir::LangItem::PanicMisalignedPointerDereference, &[required, found, location], source_info.span, ); diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs index 1b454b6667c23..50bc7a127af09 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs @@ -165,7 +165,7 @@ impl FunctionDebugContext { for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() { debug_context.dwarf.unit.line_program.row().address_offset = u64::from(start); if !loc.is_default() { - let source_loc = *self.source_loc_set.get_index(loc.bits() as usize).unwrap(); + let source_loc = self.source_loc_set[loc.bits() as usize]; create_row_for_span(debug_context, source_loc); } else { create_row_for_span(debug_context, self.function_source_loc); diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs index 3a7421d8b30d8..8a4b1cccf1460 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs @@ -38,7 +38,7 @@ pub(crate) struct DebugContext { pub(crate) struct FunctionDebugContext { entry_id: UnitEntryId, function_source_loc: (FileId, u64, u64), - source_loc_set: indexmap::IndexSet<(FileId, u64, u64)>, + source_loc_set: IndexSet<(FileId, u64, u64)>, } impl DebugContext { diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index 41e24acefbea6..3ea3884214805 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -114,9 +114,9 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { .iter() .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter()) .flatten() - .collect::>() + .collect::>() .into_iter() - .collect::>(); + .collect::>(); tcx.sess.time("codegen mono items", || { super::predefine_mono_items(tcx, &mut jit_module, &mono_items); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs index 18162fb5ab25b..fdd27a454e066 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs @@ -18,6 +18,20 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( // Spin loop hint } + // Used by is_x86_feature_detected!(); + "llvm.x86.xgetbv" => { + // FIXME use the actual xgetbv instruction + intrinsic_args!(fx, args => (v); intrinsic); + + let v = v.load_scalar(fx); + + // As of writing on XCR0 exists + fx.bcx.ins().trapnz(v, TrapCode::UnreachableCodeReached); + + let res = fx.bcx.ins().iconst(types::I64, 1 /* bit 0 must be set */); + ret.write_cvalue(fx, CValue::by_val(res, fx.layout_of(fx.tcx.types.i64))); + } + // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" @@ -53,7 +67,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32)); ret.write_cvalue(fx, res); } - "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd" => { + "llvm.x86.sse.cmp.ps" | "llvm.x86.sse2.cmp.pd" => { let (x, y, kind) = match args { [x, y, kind] => (x, y, kind), _ => bug!("wrong number of args for intrinsic {intrinsic}"), @@ -66,18 +80,95 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( let flt_cc = match kind .try_to_bits(Size::from_bytes(1)) .unwrap_or_else(|| panic!("kind not scalar: {:?}", kind)) + .try_into() + .unwrap() { - 0 => FloatCC::Equal, - 1 => FloatCC::LessThan, - 2 => FloatCC::LessThanOrEqual, - 7 => FloatCC::Ordered, - 3 => FloatCC::Unordered, - 4 => FloatCC::NotEqual, - 5 => FloatCC::UnorderedOrGreaterThanOrEqual, - 6 => FloatCC::UnorderedOrGreaterThan, + _CMP_EQ_OQ | _CMP_EQ_OS => FloatCC::Equal, + _CMP_LT_OS | _CMP_LT_OQ => FloatCC::LessThan, + _CMP_LE_OS | _CMP_LE_OQ => FloatCC::LessThanOrEqual, + _CMP_UNORD_Q | _CMP_UNORD_S => FloatCC::Unordered, + _CMP_NEQ_UQ | _CMP_NEQ_US => FloatCC::NotEqual, + _CMP_NLT_US | _CMP_NLT_UQ => FloatCC::UnorderedOrGreaterThanOrEqual, + _CMP_NLE_US | _CMP_NLE_UQ => FloatCC::UnorderedOrGreaterThan, + _CMP_ORD_Q | _CMP_ORD_S => FloatCC::Ordered, + _CMP_EQ_UQ | _CMP_EQ_US => FloatCC::UnorderedOrEqual, + _CMP_NGE_US | _CMP_NGE_UQ => FloatCC::UnorderedOrLessThan, + _CMP_NGT_US | _CMP_NGT_UQ => FloatCC::UnorderedOrLessThanOrEqual, + _CMP_FALSE_OQ | _CMP_FALSE_OS => todo!(), + _CMP_NEQ_OQ | _CMP_NEQ_OS => FloatCC::OrderedNotEqual, + _CMP_GE_OS | _CMP_GE_OQ => FloatCC::GreaterThanOrEqual, + _CMP_GT_OS | _CMP_GT_OQ => FloatCC::GreaterThan, + _CMP_TRUE_UQ | _CMP_TRUE_US => todo!(), + kind => unreachable!("kind {:?}", kind), }; + // Copied from stdarch + /// Equal (ordered, non-signaling) + const _CMP_EQ_OQ: i32 = 0x00; + /// Less-than (ordered, signaling) + const _CMP_LT_OS: i32 = 0x01; + /// Less-than-or-equal (ordered, signaling) + const _CMP_LE_OS: i32 = 0x02; + /// Unordered (non-signaling) + const _CMP_UNORD_Q: i32 = 0x03; + /// Not-equal (unordered, non-signaling) + const _CMP_NEQ_UQ: i32 = 0x04; + /// Not-less-than (unordered, signaling) + const _CMP_NLT_US: i32 = 0x05; + /// Not-less-than-or-equal (unordered, signaling) + const _CMP_NLE_US: i32 = 0x06; + /// Ordered (non-signaling) + const _CMP_ORD_Q: i32 = 0x07; + /// Equal (unordered, non-signaling) + const _CMP_EQ_UQ: i32 = 0x08; + /// Not-greater-than-or-equal (unordered, signaling) + const _CMP_NGE_US: i32 = 0x09; + /// Not-greater-than (unordered, signaling) + const _CMP_NGT_US: i32 = 0x0a; + /// False (ordered, non-signaling) + const _CMP_FALSE_OQ: i32 = 0x0b; + /// Not-equal (ordered, non-signaling) + const _CMP_NEQ_OQ: i32 = 0x0c; + /// Greater-than-or-equal (ordered, signaling) + const _CMP_GE_OS: i32 = 0x0d; + /// Greater-than (ordered, signaling) + const _CMP_GT_OS: i32 = 0x0e; + /// True (unordered, non-signaling) + const _CMP_TRUE_UQ: i32 = 0x0f; + /// Equal (ordered, signaling) + const _CMP_EQ_OS: i32 = 0x10; + /// Less-than (ordered, non-signaling) + const _CMP_LT_OQ: i32 = 0x11; + /// Less-than-or-equal (ordered, non-signaling) + const _CMP_LE_OQ: i32 = 0x12; + /// Unordered (signaling) + const _CMP_UNORD_S: i32 = 0x13; + /// Not-equal (unordered, signaling) + const _CMP_NEQ_US: i32 = 0x14; + /// Not-less-than (unordered, non-signaling) + const _CMP_NLT_UQ: i32 = 0x15; + /// Not-less-than-or-equal (unordered, non-signaling) + const _CMP_NLE_UQ: i32 = 0x16; + /// Ordered (signaling) + const _CMP_ORD_S: i32 = 0x17; + /// Equal (unordered, signaling) + const _CMP_EQ_US: i32 = 0x18; + /// Not-greater-than-or-equal (unordered, non-signaling) + const _CMP_NGE_UQ: i32 = 0x19; + /// Not-greater-than (unordered, non-signaling) + const _CMP_NGT_UQ: i32 = 0x1a; + /// False (ordered, signaling) + const _CMP_FALSE_OS: i32 = 0x1b; + /// Not-equal (ordered, signaling) + const _CMP_NEQ_OS: i32 = 0x1c; + /// Greater-than-or-equal (ordered, non-signaling) + const _CMP_GE_OQ: i32 = 0x1d; + /// Greater-than (ordered, non-signaling) + const _CMP_GT_OQ: i32 = 0x1e; + /// True (unordered, signaling) + const _CMP_TRUE_US: i32 = 0x1f; + simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| { let res_lane = match lane_ty.kind() { ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane), @@ -103,6 +194,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( _ => fx.bcx.ins().iconst(types::I32, 0), }); } + "llvm.x86.sse2.psrai.d" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.sse2.psrai.d imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } "llvm.x86.sse2.pslli.d" => { let (a, imm8) = match args { [a, imm8] => (a, imm8), @@ -137,6 +245,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( _ => fx.bcx.ins().iconst(types::I32, 0), }); } + "llvm.x86.sse2.psrai.w" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.sse2.psrai.d imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } "llvm.x86.sse2.pslli.w" => { let (a, imm8) = match args { [a, imm8] => (a, imm8), @@ -171,6 +296,57 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( _ => fx.bcx.ins().iconst(types::I32, 0), }); } + "llvm.x86.avx.psrai.d" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.avx.psrai.d imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } + "llvm.x86.sse2.psrli.q" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.avx.psrli.q imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 64 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } + "llvm.x86.sse2.pslli.q" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.avx.pslli.q imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 64 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } "llvm.x86.avx.pslli.d" => { let (a, imm8) = match args { [a, imm8] => (a, imm8), @@ -205,6 +381,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( _ => fx.bcx.ins().iconst(types::I32, 0), }); } + "llvm.x86.avx2.psrai.w" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.avx.psrai.w imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }); + } "llvm.x86.avx2.pslli.w" => { let (a, imm8) = match args { [a, imm8] => (a, imm8), @@ -313,7 +506,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( ret.place_lane(fx, 2).to_ptr().store(fx, res_2, MemFlags::trusted()); ret.place_lane(fx, 3).to_ptr().store(fx, res_3, MemFlags::trusted()); } - "llvm.x86.sse2.storeu.dq" => { + "llvm.x86.sse2.storeu.dq" | "llvm.x86.sse2.storeu.pd" => { intrinsic_args!(fx, args => (mem_addr, a); intrinsic); let mem_addr = mem_addr.load_scalar(fx); @@ -321,17 +514,45 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout()); dest.write_cvalue(fx, a); } - "llvm.x86.addcarry.64" => { + "llvm.x86.ssse3.pabs.b.128" | "llvm.x86.ssse3.pabs.w.128" | "llvm.x86.ssse3.pabs.d.128" => { + let a = match args { + [a] => a, + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { + fx.bcx.ins().iabs(lane) + }); + } + "llvm.x86.addcarry.32" | "llvm.x86.addcarry.64" => { intrinsic_args!(fx, args => (c_in, a, b); intrinsic); let c_in = c_in.load_scalar(fx); - llvm_add_sub(fx, BinOp::Add, ret, c_in, a, b); + let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b); + + let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty])); + let val = CValue::by_val_pair(cb_out, c, layout); + ret.write_cvalue(fx, val); } - "llvm.x86.subborrow.64" => { + "llvm.x86.addcarryx.u32" | "llvm.x86.addcarryx.u64" => { + intrinsic_args!(fx, args => (c_in, a, b, out); intrinsic); + let c_in = c_in.load_scalar(fx); + + let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b); + + Pointer::new(out.load_scalar(fx)).store(fx, c, MemFlags::trusted()); + ret.write_cvalue(fx, CValue::by_val(cb_out, fx.layout_of(fx.tcx.types.u8))); + } + "llvm.x86.subborrow.32" | "llvm.x86.subborrow.64" => { intrinsic_args!(fx, args => (b_in, a, b); intrinsic); let b_in = b_in.load_scalar(fx); - llvm_add_sub(fx, BinOp::Sub, ret, b_in, a, b); + let (cb_out, c) = llvm_add_sub(fx, BinOp::Sub, b_in, a, b); + + let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty])); + let val = CValue::by_val_pair(cb_out, c, layout); + ret.write_cvalue(fx, val); } _ => { fx.tcx @@ -356,21 +577,11 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( fn llvm_add_sub<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, - ret: CPlace<'tcx>, cb_in: Value, a: CValue<'tcx>, b: CValue<'tcx>, -) { - assert_eq!( - a.layout().ty, - fx.tcx.types.u64, - "llvm.x86.addcarry.64/llvm.x86.subborrow.64 second operand must be u64" - ); - assert_eq!( - b.layout().ty, - fx.tcx.types.u64, - "llvm.x86.addcarry.64/llvm.x86.subborrow.64 third operand must be u64" - ); +) -> (Value, Value) { + assert_eq!(a.layout().ty, b.layout().ty); // c + carry -> c + first intermediate carry or borrow respectively let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b); @@ -378,15 +589,14 @@ fn llvm_add_sub<'tcx>( let cb0 = int0.value_field(fx, FieldIdx::new(1)).load_scalar(fx); // c + carry -> c + second intermediate carry or borrow respectively - let cb_in_as_u64 = fx.bcx.ins().uextend(types::I64, cb_in); - let cb_in_as_u64 = CValue::by_val(cb_in_as_u64, fx.layout_of(fx.tcx.types.u64)); - let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_u64); + let clif_ty = fx.clif_type(a.layout().ty).unwrap(); + let cb_in_as_int = fx.bcx.ins().uextend(clif_ty, cb_in); + let cb_in_as_int = CValue::by_val(cb_in_as_int, fx.layout_of(a.layout().ty)); + let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_int); let (c, cb1) = int1.load_scalar_pair(fx); // carry0 | carry1 -> carry or borrow respectively let cb_out = fx.bcx.ins().bor(cb0, cb1); - let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, fx.tcx.types.u64])); - let val = CValue::by_val_pair(cb_out, c, layout); - ret.write_cvalue(fx, val); + (cb_out, c) } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index da8ab361331e7..e3006b253b7a5 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -647,12 +647,13 @@ fn codegen_regular_intrinsic_call<'tcx>( let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout); ret.write_cvalue(fx, val); } - sym::volatile_store | sym::unaligned_volatile_store => { + sym::volatile_store | sym::unaligned_volatile_store | sym::nontemporal_store => { intrinsic_args!(fx, args => (ptr, val); intrinsic); let ptr = ptr.load_scalar(fx); // Cranelift treats stores as volatile by default // FIXME correctly handle unaligned_volatile_store + // FIXME actually do nontemporal stores if requested let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); dest.write_cvalue(fx, val); } diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index 0de2dccda71f0..ebd153cb71d79 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -268,8 +268,6 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc CValue<'tcx> { } /// Load a value with layout.abi of scalar + #[track_caller] pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> Value { let layout = self.1; match self.0 { @@ -182,6 +185,7 @@ impl<'tcx> CValue<'tcx> { } /// Load a value pair with layout.abi of scalar pair + #[track_caller] pub(crate) fn load_scalar_pair(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Value, Value) { let layout = self.1; match self.0 { @@ -583,17 +587,25 @@ impl<'tcx> CPlace<'tcx> { let dst_layout = self.layout(); match self.inner { CPlaceInner::Var(_local, var) => { - let data = CValue(from.0, dst_layout).load_scalar(fx); + let data = match from.1.abi { + Abi::Scalar(_) => CValue(from.0, dst_layout).load_scalar(fx), + _ => { + let (ptr, meta) = from.force_stack(fx); + assert!(meta.is_none()); + CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar(fx) + } + }; let dst_ty = fx.clif_type(self.layout().ty).unwrap(); transmute_scalar(fx, var, data, dst_ty); } CPlaceInner::VarPair(_local, var1, var2) => { - let (data1, data2) = if from.layout().ty == dst_layout.ty { - CValue(from.0, dst_layout).load_scalar_pair(fx) - } else { - let (ptr, meta) = from.force_stack(fx); - assert!(meta.is_none()); - CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx) + let (data1, data2) = match from.1.abi { + Abi::ScalarPair(_, _) => CValue(from.0, dst_layout).load_scalar_pair(fx), + _ => { + let (ptr, meta) = from.force_stack(fx); + assert!(meta.is_none()); + CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx) + } }; let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap(); transmute_scalar(fx, var1, data1, dst_ty1); @@ -607,30 +619,38 @@ impl<'tcx> CPlace<'tcx> { let mut flags = MemFlags::new(); flags.set_notrap(); - match from.layout().abi { - Abi::Scalar(_) => { - let val = from.load_scalar(fx); - to_ptr.store(fx, val, flags); - return; - } - Abi::ScalarPair(a_scalar, b_scalar) => { - let (value, extra) = from.load_scalar_pair(fx); - let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); - to_ptr.store(fx, value, flags); - to_ptr.offset(fx, b_offset).store(fx, extra, flags); - return; - } - _ => {} - } match from.0 { CValueInner::ByVal(val) => { to_ptr.store(fx, val, flags); } - CValueInner::ByValPair(_, _) => { - bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi); - } + CValueInner::ByValPair(val1, val2) => match from.layout().abi { + Abi::ScalarPair(a_scalar, b_scalar) => { + let b_offset = + scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + to_ptr.store(fx, val1, flags); + to_ptr.offset(fx, b_offset).store(fx, val2, flags); + } + _ => bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi), + }, CValueInner::ByRef(from_ptr, None) => { + match from.layout().abi { + Abi::Scalar(_) => { + let val = from.load_scalar(fx); + to_ptr.store(fx, val, flags); + return; + } + Abi::ScalarPair(a_scalar, b_scalar) => { + let b_offset = + scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + let (val1, val2) = from.load_scalar_pair(fx); + to_ptr.store(fx, val1, flags); + to_ptr.offset(fx, b_offset).store(fx, val2, flags); + return; + } + _ => {} + } + let from_addr = from_ptr.get_addr(fx); let to_addr = to_ptr.get_addr(fx); let src_layout = from.1; @@ -815,11 +835,42 @@ pub(crate) fn assert_assignable<'tcx>( ParamEnv::reveal_all(), from_ty.fn_sig(fx.tcx), ); + let FnSig { + inputs_and_output: types_from, + c_variadic: c_variadic_from, + unsafety: unsafety_from, + abi: abi_from, + } = from_sig; let to_sig = fx .tcx .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_ty.fn_sig(fx.tcx)); + let FnSig { + inputs_and_output: types_to, + c_variadic: c_variadic_to, + unsafety: unsafety_to, + abi: abi_to, + } = to_sig; + let mut types_from = types_from.iter(); + let mut types_to = types_to.iter(); + loop { + match (types_from.next(), types_to.next()) { + (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), + (None, None) => break, + (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), + } + } + assert_eq!( + c_variadic_from, c_variadic_to, + "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", + from_sig, to_sig, fx, + ); + assert_eq!( + unsafety_from, unsafety_to, + "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", + from_sig, to_sig, fx, + ); assert_eq!( - from_sig, to_sig, + abi_from, abi_to, "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", from_sig, to_sig, fx, ); diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 2be369d2b740b..bfb967213e139 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -334,6 +334,7 @@ const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[ "cranelift-native", "cranelift-object", "crc32fast", + "equivalent", "fallible-iterator", "gimli", "hashbrown",