From e5ffb572902fe51d0f3ff5cb1bbf582f401e9c1b Mon Sep 17 00:00:00 2001 From: Tyler Mandry Date: Tue, 7 Nov 2023 18:11:07 -0800 Subject: [PATCH 1/5] Treat fuchsia like a normal git repo --- src/tools/cargotest/main.rs | 54 ++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/src/tools/cargotest/main.rs b/src/tools/cargotest/main.rs index 7044cb8928694..dd328b33f4b14 100644 --- a/src/tools/cargotest/main.rs +++ b/src/tools/cargotest/main.rs @@ -13,6 +13,8 @@ struct Test { manifest_path: Option<&'static str>, /// `filters` are passed to libtest (i.e., after a `--` in the `cargo test` invocation). filters: &'static [&'static str], + override_script: Option<&'static Path>, + run_by_default: bool, } const TEST_REPOS: &[Test] = &[ @@ -25,6 +27,8 @@ const TEST_REPOS: &[Test] = &[ features: None, manifest_path: None, filters: &[], + override_script: None, + run_by_default: true, }, Test { name: "ripgrep", @@ -35,6 +39,8 @@ const TEST_REPOS: &[Test] = &[ features: None, manifest_path: None, filters: &[], + override_script: None, + run_by_default: true, }, Test { name: "tokei", @@ -45,6 +51,8 @@ const TEST_REPOS: &[Test] = &[ features: None, manifest_path: None, filters: &[], + override_script: None, + run_by_default: true, }, Test { name: "xsv", @@ -69,6 +77,8 @@ const TEST_REPOS: &[Test] = &[ "test_stats::", "test_table::", ], + override_script: None, + run_by_default: true, }, Test { name: "servo", @@ -81,6 +91,8 @@ const TEST_REPOS: &[Test] = &[ features: None, manifest_path: None, filters: &[], + override_script: None, + run_by_default: true, }, Test { name: "diesel", @@ -97,6 +109,21 @@ const TEST_REPOS: &[Test] = &[ // (This is required to set the feature flags above) manifest_path: Some("diesel/Cargo.toml"), filters: &[], + override_script: None, + run_by_default: true, + }, + Test { + name: "fuchsia", + repo: "https://fuchsia.googlesource.com/fuchsia.git", + // TODO + sha: "refs/changes/58/938058/1", + lock: None, + packages: &[], + features: None, + manifest_path: None, + filters: &[], + override_script: Some(Path::new("scripts/rust/build_fuchsia_from_rust_ci.sh")), + run_by_default: false, }, ]; @@ -107,7 +134,10 @@ fn main() { let cargo = &Path::new(cargo); for test in TEST_REPOS.iter().rev() { - if args[3..].is_empty() || args[3..].iter().any(|s| s.contains(test.name)) { + // TODO: uninvert run_by_default below + if (args[3..].is_empty() && !test.run_by_default) + || args[3..].iter().any(|s| s.contains(test.name)) + { test_repo(cargo, out_dir, test); } } @@ -119,8 +149,18 @@ fn test_repo(cargo: &Path, out_dir: &Path, test: &Test) { if let Some(lockfile) = test.lock { fs::write(&dir.join("Cargo.lock"), lockfile).unwrap(); } - if !run_cargo_test(cargo, &dir, test.packages, test.features, test.manifest_path, test.filters) - { + let success = match test.override_script { + Some(script) => run_script(&dir, script), + None => run_cargo_test( + cargo, + &dir, + test.packages, + test.features, + test.manifest_path, + test.filters, + ), + }; + if !success { panic!("tests failed for {}", test.repo); } } @@ -148,9 +188,10 @@ fn clone_repo(test: &Test, out_dir: &Path) -> PathBuf { assert!(status.success()); } + let reset_to = if test.sha.contains('/') { "FETCH_HEAD" } else { test.sha }; let status = Command::new("git") .arg("reset") - .arg(test.sha) + .arg(reset_to) .arg("--hard") .current_dir(&out_dir) .status() @@ -165,6 +206,7 @@ fn clone_repo(test: &Test, out_dir: &Path) -> PathBuf { if !found { panic!("unable to find commit {}", test.sha) } + let status = Command::new("git").arg("clean").arg("-fdx").current_dir(&out_dir).status().unwrap(); assert!(status.success()); @@ -216,3 +258,7 @@ fn run_cargo_test( status.success() } + +fn run_script(crate_path: &Path, script: &'static Path) -> bool { + Command::new(&crate_path.join(script)).status().unwrap().success() +} From 7df62990ac75235bb97e925f52cb2f7d798e783f Mon Sep 17 00:00:00 2001 From: Tyler Mandry Date: Thu, 9 Nov 2023 17:10:39 -0800 Subject: [PATCH 2/5] wip --- .github/workflows/ci.yml | 3 + src/bootstrap/mk/Makefile.in | 9 + src/bootstrap/src/core/build_steps/compile.rs | 2 +- .../host-x86_64/dist-various-2/Dockerfile | 2 +- .../x86_64-gnu-experimental/Dockerfile | 92 ++++++++++ .../build-fuchsia-toolchain.sh | 17 +- src/ci/github-actions/ci.yml | 3 + src/tools/cargotest/main.rs | 157 ++++++++++++------ 8 files changed, 232 insertions(+), 53 deletions(-) create mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-experimental/Dockerfile rename src/ci/docker/{host-x86_64/dist-various-2 => scripts}/build-fuchsia-toolchain.sh (85%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f659a1c3060c..f9972ec3acfe1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -288,6 +288,9 @@ jobs: - name: x86_64-gnu-aux os: ubuntu-20.04-4core-16gb env: {} + - name: x86_64-gnu-experimental + os: ubuntu-20.04-16core-64gb + env: {} - name: x86_64-gnu-debug os: ubuntu-20.04-8core-32gb env: {} diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 0b67079917c7f..88544b83d8627 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -52,6 +52,15 @@ check-aux: src/tools/cargo \ src/tools/cargotest \ $(BOOTSTRAP_ARGS) +check-fuchsia: + mkdir -p $(RUST_INSTALL_DIR)/etc + $(Q)$(BOOTSTRAP) install --stage 1 \ + compiler/rustc \ + library/std + $(Q)$(BOOTSTRAP) test --stage 2 \ + src/tools/cargotest \ + --test-args=fuchsia \ + $(BOOTSTRAP_ARGS) dist: $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) distcheck: diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 7021a9543582d..92f0b0893bd73 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1711,7 +1711,7 @@ impl Step for Assemble { let dst_exe = exe("rust-lld", target_compiler.host); builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); let self_contained_lld_dir = libdir_bin.join("gcc-ld"); - t!(fs::create_dir(&self_contained_lld_dir)); + t!(fs::create_dir_all(&self_contained_lld_dir)); let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { compiler: build_compiler, target: target_compiler.host, diff --git a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile index 01b46118b9c1e..944218eb88a1e 100644 --- a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile @@ -80,7 +80,7 @@ RUN env \ WORKDIR /tmp COPY host-x86_64/dist-various-2/shared.sh /tmp/ -COPY host-x86_64/dist-various-2/build-fuchsia-toolchain.sh /tmp/ +COPY scripts/build-fuchsia-toolchain.sh /tmp/ RUN /tmp/build-fuchsia-toolchain.sh COPY host-x86_64/dist-various-2/build-solaris-toolchain.sh /tmp/ RUN /tmp/build-solaris-toolchain.sh x86_64 amd64 solaris-i386 pc diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-experimental/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-experimental/Dockerfile new file mode 100644 index 0000000000000..5b7657207077f --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-experimental/Dockerfile @@ -0,0 +1,92 @@ +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + gawk \ + g++ \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python3 \ + git \ + cmake \ + libssl-dev \ + sudo \ + time \ + xz-utils \ + pkg-config \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +ENV \ + AR_x86_64_unknown_fuchsia=x86_64-unknown-fuchsia-ar \ + CC_x86_64_unknown_fuchsia=x86_64-unknown-fuchsia-clang \ + CFLAGS_x86_64_unknown_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ + CXX_x86_64_unknown_fuchsia=x86_64-unknown-fuchsia-clang++ \ + CXXFLAGS_x86_64_unknown_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ + LDFLAGS_x86_64_unknown_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib" \ + AR_x86_64_fuchsia=x86_64-unknown-fuchsia-ar \ + CC_x86_64_fuchsia=x86_64-unknown-fuchsia-clang \ + CFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ + CXX_x86_64_fuchsia=x86_64-unknown-fuchsia-clang++ \ + CXXFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ + LDFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib" +# AR_aarch64_unknown_fuchsia=aarch64-unknown-fuchsia-ar \ +# CC_aarch64_unknown_fuchsia=aarch64-unknown-fuchsia-clang \ +# CFLAGS_aarch64_unknown_fuchsia="--target=aarch64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ +# CXX_aarch64_unknown_fuchsia=aarch64-unknown-fuchsia-clang++ \ +# CXXFLAGS_aarch64_unknown_fuchsia="--target=aarch64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ +# LDFLAGS_aarch64_unknown_fuchsia="--target=aarch64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/lib" + +WORKDIR /tmp +COPY scripts/build-fuchsia-toolchain.sh /tmp/ +RUN /tmp/build-fuchsia-toolchain.sh + +# TODO move to canonical triple +# TODO move to canonical triple +ENV CARGO_TARGET_X86_64_UNKNOWN_FUCHSIA_AR /usr/local/bin/llvm-ar +ENV CARGO_TARGET_X86_64_UNKNOWN_FUCHSIA_RUSTFLAGS \ +-C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \ +-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot/lib \ +-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib +ENV CARGO_TARGET_AARCH64_UNKNOWN_FUCHSIA_AR /usr/local/bin/llvm-ar +ENV CARGO_TARGET_AARCH64_UNKNOWN_FUCHSIA_RUSTFLAGS \ +-C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot \ +-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot/lib \ +-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/lib +ENV CARGO_TARGET_X86_64_FUCHSIA_AR /usr/local/bin/llvm-ar +ENV CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS \ +-C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \ +-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot/lib \ +-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib +ENV CARGO_TARGET_AARCH64_FUCHSIA_AR /usr/local/bin/llvm-ar +ENV CARGO_TARGET_AARCH64_FUCHSIA_RUSTFLAGS \ +-C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot \ +-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot/lib \ +-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/lib + +ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS -Clink-arg=-static-libstdc++ + +ENV TARGETS=x86_64-fuchsia +#ENV TARGETS=$TARGETS,aarch64-unknown-fuchsia +#ENV TARGETS=$TARGETS,x86_64-fuchsia +#ENV TARGETS=$TARGETS,aarch64-fuchsia +ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnu + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +#RUN mkdir -p /tmp/install/etc +ENV RUST_INSTALL_DIR /checkout/obj/install + +# --enable-extended --disable-docs +ENV RUST_CONFIGURE_ARGS --prefix=$RUST_INSTALL_DIR --sysconfdir=etc --enable-lld --llvm-libunwind=in-tree --enable-extended --disable-docs +#ENV CARGOFLAGS -Ztarget-applies-to-host +#ENV RUSTC_BOOTSTRAP 1 +ENV SCRIPT \ + python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \ + python3 ../x.py test src/tools/cargotest + diff --git a/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh b/src/ci/docker/scripts/build-fuchsia-toolchain.sh similarity index 85% rename from src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh rename to src/ci/docker/scripts/build-fuchsia-toolchain.sh index d762b4672c62e..dfd6b977fd493 100755 --- a/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh +++ b/src/ci/docker/scripts/build-fuchsia-toolchain.sh @@ -1,7 +1,22 @@ #!/usr/bin/env bash +hide_output() { + { set +x; } 2>/dev/null + on_err=" +echo ERROR: An error was encountered with the build. +cat /tmp/build.log +exit 1 +" + trap "$on_err" ERR + bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & + PING_LOOP_PID=$! + "$@" &> /tmp/build.log + trap - ERR + kill $PING_LOOP_PID + set -x +} set -ex -source shared.sh +#source shared.sh FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64 FUCHSIA_SDK_ID=4xjxrGUrDbQ6_zJwj6cDN1IbWsWV5aCQXC_zO_Hu0XkC diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index da29ffb8e5f9b..71200b59a9f0e 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -470,6 +470,9 @@ jobs: - name: x86_64-gnu-aux <<: *job-linux-4c + - name: x86_64-gnu-experimental + <<: *job-linux-16c + - name: x86_64-gnu-debug <<: *job-linux-8c diff --git a/src/tools/cargotest/main.rs b/src/tools/cargotest/main.rs index dd328b33f4b14..2f476a63a64b3 100644 --- a/src/tools/cargotest/main.rs +++ b/src/tools/cargotest/main.rs @@ -13,8 +13,6 @@ struct Test { manifest_path: Option<&'static str>, /// `filters` are passed to libtest (i.e., after a `--` in the `cargo test` invocation). filters: &'static [&'static str], - override_script: Option<&'static Path>, - run_by_default: bool, } const TEST_REPOS: &[Test] = &[ @@ -27,8 +25,6 @@ const TEST_REPOS: &[Test] = &[ features: None, manifest_path: None, filters: &[], - override_script: None, - run_by_default: true, }, Test { name: "ripgrep", @@ -39,8 +35,6 @@ const TEST_REPOS: &[Test] = &[ features: None, manifest_path: None, filters: &[], - override_script: None, - run_by_default: true, }, Test { name: "tokei", @@ -51,8 +45,6 @@ const TEST_REPOS: &[Test] = &[ features: None, manifest_path: None, filters: &[], - override_script: None, - run_by_default: true, }, Test { name: "xsv", @@ -77,8 +69,6 @@ const TEST_REPOS: &[Test] = &[ "test_stats::", "test_table::", ], - override_script: None, - run_by_default: true, }, Test { name: "servo", @@ -91,8 +81,6 @@ const TEST_REPOS: &[Test] = &[ features: None, manifest_path: None, filters: &[], - override_script: None, - run_by_default: true, }, Test { name: "diesel", @@ -109,21 +97,6 @@ const TEST_REPOS: &[Test] = &[ // (This is required to set the feature flags above) manifest_path: Some("diesel/Cargo.toml"), filters: &[], - override_script: None, - run_by_default: true, - }, - Test { - name: "fuchsia", - repo: "https://fuchsia.googlesource.com/fuchsia.git", - // TODO - sha: "refs/changes/58/938058/1", - lock: None, - packages: &[], - features: None, - manifest_path: None, - filters: &[], - override_script: Some(Path::new("scripts/rust/build_fuchsia_from_rust_ci.sh")), - run_by_default: false, }, ]; @@ -133,12 +106,16 @@ fn main() { let out_dir = Path::new(&args[2]); let cargo = &Path::new(cargo); - for test in TEST_REPOS.iter().rev() { - // TODO: uninvert run_by_default below - if (args[3..].is_empty() && !test.run_by_default) - || args[3..].iter().any(|s| s.contains(test.name)) - { - test_repo(cargo, out_dir, test); + if false { + for test in TEST_REPOS.iter().rev() { + if args[3..].is_empty() || args[3..].iter().any(|s| s.contains(test.name)) { + test_repo(cargo, out_dir, test); + } + } + } else { + // For now, let Fuchsia tests fail. + if std::panic::catch_unwind(|| test_fuchsia(out_dir)).is_err() { + eprintln!("Fuchsia tests failed; continuing."); } } } @@ -149,18 +126,8 @@ fn test_repo(cargo: &Path, out_dir: &Path, test: &Test) { if let Some(lockfile) = test.lock { fs::write(&dir.join("Cargo.lock"), lockfile).unwrap(); } - let success = match test.override_script { - Some(script) => run_script(&dir, script), - None => run_cargo_test( - cargo, - &dir, - test.packages, - test.features, - test.manifest_path, - test.filters, - ), - }; - if !success { + if !run_cargo_test(cargo, &dir, test.packages, test.features, test.manifest_path, test.filters) + { panic!("tests failed for {}", test.repo); } } @@ -188,10 +155,9 @@ fn clone_repo(test: &Test, out_dir: &Path) -> PathBuf { assert!(status.success()); } - let reset_to = if test.sha.contains('/') { "FETCH_HEAD" } else { test.sha }; let status = Command::new("git") .arg("reset") - .arg(reset_to) + .arg(test.sha) .arg("--hard") .current_dir(&out_dir) .status() @@ -206,7 +172,6 @@ fn clone_repo(test: &Test, out_dir: &Path) -> PathBuf { if !found { panic!("unable to find commit {}", test.sha) } - let status = Command::new("git").arg("clean").arg("-fdx").current_dir(&out_dir).status().unwrap(); assert!(status.success()); @@ -259,6 +224,98 @@ fn run_cargo_test( status.success() } -fn run_script(crate_path: &Path, script: &'static Path) -> bool { - Command::new(&crate_path.join(script)).status().unwrap().success() +fn test_fuchsia(out_dir: &Path) { + const INTEGRATION_SHA: &str = "06ae16d18bd8e4db9a3fc062f678a170025d9f1a"; + const PICK_REFS: &[&str] = &["refs/changes/58/938058/3", "refs/changes/33/943833/5"]; + + // This script will: + // - create a directory named "fuchsia" if it does not exist + // - download "jiri" to "fuchsia/.jiri_root/bin" + const BOOTSTRAP: &'static str = r#" + curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" | base64 --decode | bash -s fuchsia + "#; + let status = Command::new("sh").arg("-c").arg(BOOTSTRAP).current_dir(out_dir).status().unwrap(); + assert!(status.success(), "bootstrap_jiri failed"); + + let checkout_dir = &out_dir.join("fuchsia"); + let jiri = || { + let mut cmd = Command::new(".jiri_root/bin/jiri"); + cmd.current_dir(checkout_dir); + cmd + }; + + let status = jiri() + .arg("init") + .arg("-partial=true") + .arg("-analytics-opt=false") + .arg(checkout_dir) + .status() + .unwrap(); + assert!(status.success(), "jiri init failed"); + + let status = jiri() + .args([ + "import", + "-name=integration", + &format!("-revision={INTEGRATION_SHA}"), + "-overwrite=true", + "flower", + "https://fuchsia.googlesource.com/integration", + ]) + .status() + .unwrap(); + assert!(status.success(), "jiri import failed"); + + if checkout_dir.join(".git").is_dir() { + // Wipe out any local changes if we're reusing a checkout. + let status = Command::new("git") + .arg("checkout") + .arg("--force") + .arg("JIRI_HEAD") + .current_dir(checkout_dir) + .status() + .unwrap(); + assert!(status.success(), "checkout JIRI_HEAD failed"); + } + + let status = jiri() + .arg("update") + .arg("-autoupdate=false") + // .arg("-v") + .status() + .unwrap(); + assert!(status.success(), "jiri update failed"); + + let integration = Command::new("git") + .current_dir(checkout_dir.join("integration")) + .arg("rev-parse") + .arg("HEAD") + .output() + .unwrap() + .stdout; + let integration = String::from_utf8_lossy(&integration); + println!("integration commit = {integration}"); + + for git_ref in PICK_REFS { + let status = Command::new("git") + .current_dir(checkout_dir) + .args(["fetch", "https://fuchsia.googlesource.com/fuchsia", git_ref]) + .status() + .unwrap(); + assert!(status.success(), "fetching ref '{git_ref}' failed"); + + let status = Command::new("git") + .current_dir(checkout_dir) + .args(["cherry-pick", "--no-commit", "FETCH_HEAD"]) + .status() + .unwrap(); + assert!(status.success(), "picking ref '{git_ref}' failed"); + } + + let status = Command::new("bash") + .current_dir(checkout_dir) + .arg("scripts/rust/build_fuchsia_from_rust_ci.sh") + .status() + .unwrap(); + assert!(status.success(), "fuchsia build failed"); } From 90a6f443f086e46129e6ae57251381f0ee8036b9 Mon Sep 17 00:00:00 2001 From: Tyler Mandry Date: Thu, 9 Nov 2023 18:27:17 -0800 Subject: [PATCH 3/5] Bump Fuchsia clang and SDK --- src/ci/docker/scripts/build-fuchsia-toolchain.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ci/docker/scripts/build-fuchsia-toolchain.sh b/src/ci/docker/scripts/build-fuchsia-toolchain.sh index dfd6b977fd493..f0c7189502f75 100755 --- a/src/ci/docker/scripts/build-fuchsia-toolchain.sh +++ b/src/ci/docker/scripts/build-fuchsia-toolchain.sh @@ -19,13 +19,13 @@ set -ex #source shared.sh FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64 -FUCHSIA_SDK_ID=4xjxrGUrDbQ6_zJwj6cDN1IbWsWV5aCQXC_zO_Hu0XkC -FUCHSIA_SDK_SHA256=e318f1ac652b0db43aff32708fa70337521b5ac595e5a0905c2ff33bf1eed179 +FUCHSIA_SDK_ID=MrhQwtmP8CpZre-i_PNOREcThbUcrX3bA-45d6WQr-cC +FUCHSIA_SDK_SHA256=32b850c2d98ff02a59adefa2fcf34e44471385b51cad7ddb03ee3977a590afe7 FUCHSIA_SDK_USR_DIR=/usr/local/core-linux-amd64-fuchsia-sdk CLANG_DOWNLOAD_URL=\ https://chrome-infra-packages.appspot.com/dl/fuchsia/third_party/clang/linux-amd64 -CLANG_DOWNLOAD_ID=vU0vNjSihOV4Q6taQYCpy03JXGiCyVwxen3rFMNMIgsC -CLANG_DOWNLOAD_SHA256=bd4d2f3634a284e57843ab5a4180a9cb4dc95c6882c95c317a7deb14c34c220b +CLANG_DOWNLOAD_ID=Tpc85d1ZwSlZ6UKl2d96GRUBGNA5JKholOKe24sRDr0C +CLANG_DOWNLOAD_SHA256=4e973ce5dd59c12959e942a5d9df7a19150118d03924a86894e29edb8b110ebd install_clang() { mkdir -p clang_download From 731c53571db77ba4fc74f61b988b1e80601b9a1e Mon Sep 17 00:00:00 2001 From: Tyler Mandry Date: Thu, 9 Nov 2023 18:27:32 -0800 Subject: [PATCH 4/5] cleanup --- .../x86_64-gnu-experimental/Dockerfile | 27 +++---------------- .../docker/scripts/build-fuchsia-toolchain.sh | 2 ++ src/tools/cargotest/main.rs | 2 +- 3 files changed, 6 insertions(+), 25 deletions(-) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-experimental/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-experimental/Dockerfile index 5b7657207077f..429be0e1610f0 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-experimental/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-experimental/Dockerfile @@ -21,6 +21,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ unzip \ && rm -rf /var/lib/apt/lists/* +# Copied from dist-various-2 Dockerfile. +# TODO move to canonical triple and deduplicate this list ENV \ AR_x86_64_unknown_fuchsia=x86_64-unknown-fuchsia-ar \ CC_x86_64_unknown_fuchsia=x86_64-unknown-fuchsia-clang \ @@ -34,29 +36,12 @@ ENV \ CXX_x86_64_fuchsia=x86_64-unknown-fuchsia-clang++ \ CXXFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ LDFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib" -# AR_aarch64_unknown_fuchsia=aarch64-unknown-fuchsia-ar \ -# CC_aarch64_unknown_fuchsia=aarch64-unknown-fuchsia-clang \ -# CFLAGS_aarch64_unknown_fuchsia="--target=aarch64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ -# CXX_aarch64_unknown_fuchsia=aarch64-unknown-fuchsia-clang++ \ -# CXXFLAGS_aarch64_unknown_fuchsia="--target=aarch64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ -# LDFLAGS_aarch64_unknown_fuchsia="--target=aarch64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/lib" WORKDIR /tmp COPY scripts/build-fuchsia-toolchain.sh /tmp/ RUN /tmp/build-fuchsia-toolchain.sh # TODO move to canonical triple -# TODO move to canonical triple -ENV CARGO_TARGET_X86_64_UNKNOWN_FUCHSIA_AR /usr/local/bin/llvm-ar -ENV CARGO_TARGET_X86_64_UNKNOWN_FUCHSIA_RUSTFLAGS \ --C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \ --Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot/lib \ --Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib -ENV CARGO_TARGET_AARCH64_UNKNOWN_FUCHSIA_AR /usr/local/bin/llvm-ar -ENV CARGO_TARGET_AARCH64_UNKNOWN_FUCHSIA_RUSTFLAGS \ --C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot \ --Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot/lib \ --Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/lib ENV CARGO_TARGET_X86_64_FUCHSIA_AR /usr/local/bin/llvm-ar ENV CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS \ -C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \ @@ -71,21 +56,15 @@ ENV CARGO_TARGET_AARCH64_FUCHSIA_RUSTFLAGS \ ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS -Clink-arg=-static-libstdc++ ENV TARGETS=x86_64-fuchsia -#ENV TARGETS=$TARGETS,aarch64-unknown-fuchsia -#ENV TARGETS=$TARGETS,x86_64-fuchsia -#ENV TARGETS=$TARGETS,aarch64-fuchsia ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnu COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh -#RUN mkdir -p /tmp/install/etc ENV RUST_INSTALL_DIR /checkout/obj/install +RUN mkdir -p $RUST_INSTALL_DIR/etc -# --enable-extended --disable-docs ENV RUST_CONFIGURE_ARGS --prefix=$RUST_INSTALL_DIR --sysconfdir=etc --enable-lld --llvm-libunwind=in-tree --enable-extended --disable-docs -#ENV CARGOFLAGS -Ztarget-applies-to-host -#ENV RUSTC_BOOTSTRAP 1 ENV SCRIPT \ python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \ python3 ../x.py test src/tools/cargotest diff --git a/src/ci/docker/scripts/build-fuchsia-toolchain.sh b/src/ci/docker/scripts/build-fuchsia-toolchain.sh index f0c7189502f75..146c4a5784d81 100755 --- a/src/ci/docker/scripts/build-fuchsia-toolchain.sh +++ b/src/ci/docker/scripts/build-fuchsia-toolchain.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash + +# TODO: copied from shared.sh hide_output() { { set +x; } 2>/dev/null on_err=" diff --git a/src/tools/cargotest/main.rs b/src/tools/cargotest/main.rs index 2f476a63a64b3..d8036c1f14891 100644 --- a/src/tools/cargotest/main.rs +++ b/src/tools/cargotest/main.rs @@ -226,7 +226,7 @@ fn run_cargo_test( fn test_fuchsia(out_dir: &Path) { const INTEGRATION_SHA: &str = "06ae16d18bd8e4db9a3fc062f678a170025d9f1a"; - const PICK_REFS: &[&str] = &["refs/changes/58/938058/3", "refs/changes/33/943833/5"]; + const PICK_REFS: &[&str] = &["refs/changes/58/938058/3", "refs/changes/33/943833/6"]; // This script will: // - create a directory named "fuchsia" if it does not exist From 234ade59ca3a6e1de1ab574785ce2659996f1ded Mon Sep 17 00:00:00 2001 From: Tyler Mandry Date: Thu, 9 Nov 2023 19:16:07 -0800 Subject: [PATCH 5/5] Add builder to try --- .github/workflows/ci.yml | 3 +++ src/ci/github-actions/ci.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9972ec3acfe1..825080ba19e9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -586,6 +586,9 @@ jobs: env: CODEGEN_BACKENDS: "llvm,cranelift" os: ubuntu-20.04-16core-64gb + - name: x86_64-gnu-experimental + os: ubuntu-20.04-16core-64gb + env: {} timeout-minutes: 600 runs-on: "${{ matrix.os }}" steps: diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 71200b59a9f0e..579e94a86e560 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -737,6 +737,9 @@ jobs: env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c + - name: x86_64-gnu-experimental + <<: *job-linux-16c + master: name: master