From c2354b9a113a00b802ccca7e5fa8c548c97b8577 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 18 Jul 2019 08:35:25 -0700 Subject: [PATCH] Setup CI with Azure Pipelines --- .travis.yml | 75 --------------------------- README.md | 3 +- appveyor.yml | 22 -------- azure-pipelines.yml | 91 +++++++++++++++++++++++++++++++++ ci/azure-install-rust.yml | 28 +++++++++++ ci/azure-test-all.yml | 28 +++++++++++ tests/testsuite/config.rs | 8 ++- tests/testsuite/support/mod.rs | 92 +++++++++++++++++++++------------- tests/testsuite/tool_paths.rs | 10 ++-- 9 files changed, 220 insertions(+), 137 deletions(-) delete mode 100644 .travis.yml delete mode 100644 appveyor.yml create mode 100644 azure-pipelines.yml create mode 100644 ci/azure-install-rust.yml create mode 100644 ci/azure-test-all.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 020002e5a26..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,75 +0,0 @@ -language: rust -rust: stable -dist: trusty - -git: - depth: 1 - -matrix: - include: - - name: "rustfmt" - env: TARGET=x86_64-unknown-linux-gnu - rust: stable - addons: - before_script: - - rustup component add rustfmt - script: - - cargo fmt --all -- --check - - cd crates/cargo-test-macro - - cargo fmt --all -- --check - - cd ../crates-io - - cargo fmt --all -- --check - - cd ../resolver-tests - - cargo fmt --all -- --check - - cd ../../ - - - env: TARGET=x86_64-unknown-linux-gnu - ALT=i686-unknown-linux-gnu - if: branch != master OR type = pull_request - - - env: TARGET=x86_64-apple-darwin - ALT=i686-apple-darwin - os: osx - osx_image: xcode9.2 - if: branch != master OR type = pull_request - - - env: TARGET=x86_64-unknown-linux-gnu - ALT=i686-unknown-linux-gnu - rust: beta - if: branch != master OR type = pull_request - - - env: TARGET=x86_64-unknown-linux-gnu - ALT=i686-unknown-linux-gnu - rust: nightly - install: - - travis_retry curl -Lf https://github.com/rust-lang-nursery/mdBook/releases/download/v0.3.1/mdbook-v0.3.1-x86_64-unknown-linux-gnu.tar.gz | tar -xz --directory=$HOME/.cargo/bin - script: - - cargo test --features=deny-warnings || travis_terminate 1 - - cargo doc --no-deps || travis_terminate 1 - - (cd src/doc && mdbook build --dest-dir ../../target/doc) || travis_terminate 1 - if: branch != master OR type = pull_request - - - name: resolver tests - rust: stable - before_script: true - script: - - cargo test --manifest-path crates/resolver-tests/Cargo.toml - if: branch != master OR type = pull_request - - exclude: - - rust: stable - -before_script: - - rustup target add $ALT - - rustup component add clippy || echo "clippy not available" -script: - - cargo test --features=deny-warnings - -notifications: - email: - on_success: never - -addons: - apt: - packages: - - gcc-multilib diff --git a/README.md b/README.md index 9bf4fa779d0..813c44f4e28 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,7 @@ Learn more at https://doc.rust-lang.org/cargo/ ## Code Status -[![Build Status](https://travis-ci.com/rust-lang/cargo.svg?branch=master)](https://travis-ci.com/rust-lang/cargo) -[![Build Status](https://ci.appveyor.com/api/projects/status/github/rust-lang/cargo?branch=master&svg=true)](https://ci.appveyor.com/project/rust-lang-libs/cargo) +[![Build Status](https://dev.azure.com/rust-lang/cargo/_apis/build/status/rust-lang.cargo?branchName=master)](https://dev.azure.com/rust-lang/cargo/_build/latest?definitionId=18&branchName=master) Code documentation: https://docs.rs/cargo/ diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 4173501e3a0..00000000000 --- a/appveyor.yml +++ /dev/null @@ -1,22 +0,0 @@ -environment: - matrix: - - TARGET: x86_64-pc-windows-msvc - OTHER_TARGET: i686-pc-windows-msvc - -install: - - if NOT defined APPVEYOR_PULL_REQUEST_NUMBER if "%APPVEYOR_REPO_BRANCH%" == "master" appveyor exit - - appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe - - rustup-init.exe -y --default-host x86_64-pc-windows-msvc --default-toolchain nightly - - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin - - if defined OTHER_TARGET rustup target add %OTHER_TARGET% - - rustup component add clippy || exit 0 - - rustc -V - - cargo -V - - git submodule update --init - -clone_depth: 1 - -build: false - -test_script: - - cargo test --features=deny-warnings diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000000..5bb3f65094f --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,91 @@ +trigger: + branches: + include: + - '*' + exclude: + - master +pr: +- master + +jobs: +- job: Linux + pool: + vmImage: ubuntu-16.04 + steps: + - template: ci/azure-test-all.yml + strategy: + matrix: + stable: + TOOLCHAIN: stable + beta: + TOOLCHAIN: beta + nightly: + TOOLCHAIN: nightly + variables: + OTHER_TARGET: i686-unknown-linux-gnu + +- job: macOS + pool: + vmImage: macos-10.13 + steps: + - template: ci/azure-test-all.yml + variables: + TOOLCHAIN: stable + OTHER_TARGET: i686-apple-darwin + +- job: Windows + pool: + vmImage: windows-2019 + steps: + - template: ci/azure-test-all.yml + strategy: + matrix: + x86_64-msvc: + TOOLCHAIN: stable-x86_64-pc-windows-msvc + OTHER_TARGET: i686-pc-windows-msvc +- job: rustfmt + pool: + vmImage: ubuntu-16.04 + steps: + - template: ci/azure-install-rust.yml + - bash: rustup component add rustfmt + displayName: "Install rustfmt" + - bash: cargo fmt --all -- --check + displayName: "Check rustfmt (cargo)" + - bash: cd crates/cargo-test-macro && cargo fmt --all -- --check + displayName: "Check rustfmt (cargo-test-macro)" + - bash: cd crates/crates-io && cargo fmt --all -- --check + displayName: "Check rustfmt (crates-io)" + - bash: cd crates/resolver-tests && cargo fmt --all -- --check + displayName: "Check rustfmt (resolver-tests)" + variables: + TOOLCHAIN: stable + +- job: resolver + pool: + vmImage: ubuntu-16.04 + steps: + - template: ci/azure-install-rust.yml + - bash: cargo test --manifest-path crates/resolver-tests/Cargo.toml + displayName: "Resolver tests" + variables: + TOOLCHAIN: stable + +- job: docs + pool: + vmImage: ubuntu-16.04 + steps: + - template: ci/azure-install-rust.yml + - bash: | + set -e + mkdir mdbook + curl -Lf https://github.com/rust-lang-nursery/mdBook/releases/download/v0.3.1/mdbook-v0.3.1-x86_64-unknown-linux-gnu.tar.gz | tar -xz --directory=./mdbook + echo "##vso[task.prependpath]`pwd`/mdbook" + displayName: "Install mdbook" + - bash: cargo doc --no-deps + displayName: "Build documentation" + - bash: cd src/doc && mdbook build --dest-dir ../../target/doc + displayName: "Build mdbook documentation" + variables: + TOOLCHAIN: stable + diff --git a/ci/azure-install-rust.yml b/ci/azure-install-rust.yml new file mode 100644 index 00000000000..c48d0d0155f --- /dev/null +++ b/ci/azure-install-rust.yml @@ -0,0 +1,28 @@ +steps: + - bash: | + set -e + if command -v rustup; then + echo `command -v rustup` `rustup -V` already installed + rustup self update + elif [ "$AGENT_OS" = "Windows_NT" ]; then + curl -sSf -o rustup-init.exe https://win.rustup.rs + rustup-init.exe -y --default-toolchain $TOOLCHAIN + echo "##vso[task.prependpath]$USERPROFILE/.cargo/bin" + else + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $TOOLCHAIN + echo "##vso[task.prependpath]$HOME/.cargo/bin" + fi + displayName: Install rustup + + - bash: | + set -e + rustup update $TOOLCHAIN + rustup default $TOOLCHAIN + displayName: Install rust + + - bash: | + set -ex + rustup -V + rustc -Vv + cargo -V + displayName: Query rust and cargo versions diff --git a/ci/azure-test-all.yml b/ci/azure-test-all.yml new file mode 100644 index 00000000000..626858431e8 --- /dev/null +++ b/ci/azure-test-all.yml @@ -0,0 +1,28 @@ +steps: +- checkout: self + fetchDepth: 1 + +- template: azure-install-rust.yml + +- bash: rustup target add $OTHER_TARGET + displayName: "Install cross-compile target" + +- bash: sudo apt install gcc-multilib + displayName: "Install gcc-multilib (linux)" + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + +# Some tests rely on a clippy command to run, so let's try to install clippy to +# we can be sure to run those tests. +- bash: rustup component add clippy || echo "clippy not available" + displayName: "Install clippy (maybe)" + +# Deny warnings on CI to avoid warnings getting into the codebase, and note the +# `force-system-lib-on-osx` which is intended to fix compile issues on OSX where +# compiling curl from source on OSX yields linker errors on Azure. +# +# Note that the curl issue is traced back to alexcrichton/curl-rust#279 where it +# looks like the OSX version we're actually running on is such that a symbol is +# emitted that's never worked. For now force the system library to be used to +# fix the link errors. +- bash: cargo test --features 'deny-warnings curl/force-system-lib-on-osx' + displayName: "cargo test" diff --git a/tests/testsuite/config.rs b/tests/testsuite/config.rs index d54bc64dbea..01f79271a92 100644 --- a/tests/testsuite/config.rs +++ b/tests/testsuite/config.rs @@ -2,12 +2,18 @@ use std::borrow::Borrow; use std::collections; use std::fs; -use crate::support::{lines_match, paths, project}; +use crate::support::{paths, project}; use cargo::core::{enable_nightly_features, Shell}; use cargo::util::config::{self, Config}; use cargo::util::toml::{self, VecStringOrBool as VSOB}; use serde::Deserialize; +fn lines_match(a: &str, b: &str) -> bool { + // Perform a small amount of normalization for filesystem paths before we + // send this to the `lines_match` function. + crate::support::lines_match(&a.replace("\\", "/"), &b.replace("\\", "/")) +} + #[cargo_test] fn read_env_vars_for_config() { let p = project() diff --git a/tests/testsuite/support/mod.rs b/tests/testsuite/support/mod.rs index 745dc871d8c..ac7c55a3673 100644 --- a/tests/testsuite/support/mod.rs +++ b/tests/testsuite/support/mod.rs @@ -578,8 +578,8 @@ pub struct Execs { expect_stderr_unordered: Vec, expect_neither_contains: Vec, expect_stderr_with_without: Vec<(Vec, Vec)>, - expect_json: Option>, - expect_json_contains_unordered: Vec, + expect_json: Option>, + expect_json_contains_unordered: Vec, stream_output: bool, } @@ -746,7 +746,7 @@ impl Execs { self.expect_json = Some( expected .split("\n\n") - .map(|line| line.parse().expect("line to be a valid JSON value")) + .map(|line| line.to_string()) .collect(), ); self @@ -762,11 +762,8 @@ impl Execs { /// /// See `with_json` for more detail. pub fn with_json_contains_unordered(&mut self, expected: &str) -> &mut Self { - self.expect_json_contains_unordered.extend( - expected - .split("\n\n") - .map(|line| line.parse().expect("line to be a valid JSON value")), - ); + self.expect_json_contains_unordered + .extend(expected.split("\n\n").map(|line| line.to_string())); self } @@ -1110,25 +1107,51 @@ impl Execs { Err(..) => return Err(format!("{} was not utf8 encoded", description)), Ok(actual) => actual, }; - // Let's not deal with \r\n vs \n on windows... - let actual = actual.replace("\r", ""); - let actual = actual.replace("\t", ""); - Ok(actual) - } + Ok(self.normalize_matcher(actual)) + } + + fn normalize_matcher(&self, matcher: &str) -> String { + // Let's not deal with / vs \ (windows...) + let matcher = matcher.replace("\\\\", "/").replace("\\", "/"); + + // Weirdness for paths on Windows extends beyond `/` vs `\` apparently. + // Namely paths like `c:\` and `C:\` are equivalent and that can cause + // issues. The return value of `env::current_dir()` may return a + // lowercase drive name, but we round-trip a lot of values through `Url` + // which will auto-uppercase the drive name. To just ignore this + // distinction we try to canonicalize as much as possible, taking all + // forms of a path and canonicalizing them to one. + let replace_path = |s: &str, path: &Path, with: &str| { + let path_through_url = Url::from_file_path(path).unwrap().to_file_path().unwrap(); + let path1 = path.display().to_string().replace("\\", "/"); + let path2 = path_through_url.display().to_string().replace("\\", "/"); + s.replace(&path1, with) + .replace(&path2, with) + .replace(with, &path1) + }; - fn replace_expected(&self, expected: &str) -> String { // Do the template replacements on the expected string. - let replaced = match self.process_builder { - None => expected.to_string(), - Some(ref p) => match p.get_cwd() { - None => expected.to_string(), - Some(cwd) => expected.replace("[CWD]", &cwd.display().to_string()), + let matcher = match &self.process_builder { + None => matcher.to_string(), + Some(p) => match p.get_cwd() { + None => matcher.to_string(), + Some(cwd) => replace_path(&matcher, cwd, "[CWD]"), }, }; - // On Windows, we need to use a wildcard for the drive, - // because we don't actually know what it will be. - replaced.replace("[ROOT]", if cfg!(windows) { r#"[..]:\"# } else { "/" }) + // Similar to cwd above, perform similar treatment to the root path + // which in theory all of our paths should otherwise get rooted at. + let root = paths::root(); + let matcher = replace_path(&matcher, &root, "[ROOT]"); + + // Let's not deal with \r\n vs \n on windows... + let matcher = matcher.replace("\r", ""); + + // It's easier to read tabs in outputs if they don't show up as literal + // hidden characters + let matcher = matcher.replace("\t", ""); + + return matcher; } fn match_std( @@ -1140,7 +1163,7 @@ impl Execs { kind: MatchKind, ) -> MatchResult { let out = match expected { - Some(out) => self.replace_expected(out), + Some(out) => self.normalize_matcher(out), None => return Ok(()), }; @@ -1276,7 +1299,7 @@ impl Execs { ) -> MatchResult { let actual = self.normalize_actual("stderr", actual)?; let contains = |s, line| { - let mut s = self.replace_expected(s); + let mut s = self.normalize_matcher(s); s.insert_str(0, "[..]"); s.push_str("[..]"); lines_match(&s, line) @@ -1309,13 +1332,19 @@ impl Execs { } } - fn match_json(&self, expected: &Value, line: &str) -> MatchResult { + fn match_json(&self, expected: &str, line: &str) -> MatchResult { + let expected = self.normalize_matcher(expected); + let line = self.normalize_matcher(line); let actual = match line.parse() { Err(e) => return Err(format!("invalid json, {}:\n`{}`", e, line)), Ok(actual) => actual, }; + let expected = match expected.parse() { + Err(e) => return Err(format!("invalid json, {}:\n`{}`", e, line)), + Ok(expected) => expected, + }; - find_json_mismatch(expected, &actual) + find_json_mismatch(&expected, &actual) } fn diff_lines<'a>( @@ -1372,14 +1401,9 @@ enum MatchKind { /// - There is a wide range of macros (such as `[COMPILING]` or `[WARNING]`) /// to match cargo's "status" output and allows you to ignore the alignment. /// See `substitute_macros` for a complete list of macros. -/// - `[ROOT]` is `/` or `[..]:\` on Windows. +/// - `[ROOT]` the path to the test directory's root /// - `[CWD]` is the working directory of the process that was run. -pub fn lines_match(expected: &str, actual: &str) -> bool { - // Let's not deal with / vs \ (windows...) - // First replace backslash-escaped backslashes with forward slashes - // which can occur in, for example, JSON output - let expected = expected.replace("\\\\", "/").replace("\\", "/"); - let mut actual: &str = &actual.replace("\\\\", "/").replace("\\", "/"); +pub fn lines_match(expected: &str, mut actual: &str) -> bool { let expected = substitute_macros(&expected); for (i, part) in expected.split("[..]").enumerate() { match actual.find(part) { @@ -1742,7 +1766,7 @@ pub fn is_coarse_mtime() -> bool { // This should actually be a test that `$CARGO_TARGET_DIR` is on an HFS // filesystem, (or any filesystem with low-resolution mtimes). However, // that's tricky to detect, so for now just deal with CI. - cfg!(target_os = "macos") && env::var("CI").is_ok() + cfg!(target_os = "macos") && (env::var("CI").is_ok() || env::var("TF_BUILD").is_ok()) } /// Some CI setups are much slower then the equipment used by Cargo itself. diff --git a/tests/testsuite/tool_paths.rs b/tests/testsuite/tool_paths.rs index 8633871e5c0..07dfa082999 100644 --- a/tests/testsuite/tool_paths.rs +++ b/tests/testsuite/tool_paths.rs @@ -64,11 +64,15 @@ fn absolute_tools() { ) .build(); - foo.cargo("build --verbose").with_stderr("\ + foo.cargo("build --verbose") + .with_stderr( + "\ [COMPILING] foo v0.5.0 ([CWD]) -[RUNNING] `rustc [..] -C ar=[ROOT]bogus/nonexistent-ar -C linker=[ROOT]bogus/nonexistent-linker [..]` +[RUNNING] `rustc [..] -C ar=[..]bogus/nonexistent-ar -C linker=[..]bogus/nonexistent-linker [..]` [FINISHED] dev [unoptimized + debuginfo] target(s) in [..] -").run(); +", + ) + .run(); } #[cargo_test]