From 4cb9ac35bf24bf95a8a09db78d0eccbb299e9f8b Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Sat, 14 Jan 2023 14:48:53 -0800 Subject: [PATCH] Add network container tests --- .github/workflows/main.yml | 4 + crates/cargo-test-macro/src/lib.rs | 17 + crates/cargo-test-support/build.rs | 1 + .../containers/apache/Dockerfile | 26 + .../containers/apache/bar/Cargo.toml | 4 + .../containers/apache/bar/src/lib.rs | 1 + .../containers/apache/httpd-cargo.conf | 12 + .../containers/sshd/Dockerfile | 29 + .../containers/sshd/bar/Cargo.toml | 4 + .../containers/sshd/bar/src/lib.rs | 1 + crates/cargo-test-support/src/containers.rs | 285 +++++++++ crates/cargo-test-support/src/lib.rs | 7 + src/cargo/sources/git/known_hosts.rs | 3 +- src/cargo/util/network.rs | 2 +- src/doc/contrib/src/tests/running.md | 27 + src/doc/contrib/src/tests/writing.md | 13 + tests/testsuite/https.rs | 152 +++++ tests/testsuite/main.rs | 2 + tests/testsuite/ssh.rs | 548 ++++++++++++++++++ 19 files changed, 1136 insertions(+), 2 deletions(-) create mode 100644 crates/cargo-test-support/containers/apache/Dockerfile create mode 100644 crates/cargo-test-support/containers/apache/bar/Cargo.toml create mode 100644 crates/cargo-test-support/containers/apache/bar/src/lib.rs create mode 100644 crates/cargo-test-support/containers/apache/httpd-cargo.conf create mode 100644 crates/cargo-test-support/containers/sshd/Dockerfile create mode 100644 crates/cargo-test-support/containers/sshd/bar/Cargo.toml create mode 100644 crates/cargo-test-support/containers/sshd/bar/src/lib.rs create mode 100644 crates/cargo-test-support/src/containers.rs create mode 100644 tests/testsuite/https.rs create mode 100644 tests/testsuite/ssh.rs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f9b09ff6868..ea482e82360 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,6 +34,7 @@ jobs: CARGO_PROFILE_DEV_DEBUG: 1 CARGO_PROFILE_TEST_DEBUG: 1 CARGO_INCREMENTAL: 0 + CARGO_PUBLIC_NETWORK_TESTS: 1 strategy: matrix: include: @@ -77,6 +78,9 @@ jobs: - run: sudo apt update -y && sudo apt install gcc-multilib libsecret-1-0 libsecret-1-dev -y if: matrix.os == 'ubuntu-latest' - run: rustup component add rustfmt || echo "rustfmt not available" + - name: Configure extra test environment + run: echo CARGO_CONTAINER_TESTS=1 >> $GITHUB_ENV + if: matrix.os == 'ubuntu-latest' # Deny warnings on CI to avoid warnings getting into the codebase. - run: cargo test --features 'deny-warnings' diff --git a/crates/cargo-test-macro/src/lib.rs b/crates/cargo-test-macro/src/lib.rs index 40661ca5119..c3ac95df3b3 100644 --- a/crates/cargo-test-macro/src/lib.rs +++ b/crates/cargo-test-macro/src/lib.rs @@ -55,6 +55,23 @@ pub fn cargo_test(attr: TokenStream, item: TokenStream) -> TokenStream { "does not work on windows-gnu" ); } + "container_test" => { + // These tests must be opt-in because they require docker. + set_ignore!( + option_env!("CARGO_CONTAINER_TESTS").is_none(), + "CARGO_CONTAINER_TESTS must be set" + ); + } + "public_network_test" => { + // These tests must be opt-in because they touch the public + // network. The use of these should be **EXTREMELY RARE**, and + // should only touch things which would nearly certainly work + // in CI (like github.com). + set_ignore!( + option_env!("CARGO_PUBLIC_NETWORK_TESTS").is_none(), + "CARGO_PUBLIC_NETWORK_TESTS must be set" + ); + } "nightly" => { requires_reason = true; set_ignore!(is_not_nightly, "requires nightly"); diff --git a/crates/cargo-test-support/build.rs b/crates/cargo-test-support/build.rs index 4519defc356..478da7d99f3 100644 --- a/crates/cargo-test-support/build.rs +++ b/crates/cargo-test-support/build.rs @@ -3,4 +3,5 @@ fn main() { "cargo:rustc-env=NATIVE_ARCH={}", std::env::var("TARGET").unwrap() ); + println!("cargo:rerun-if-changed=build.rs"); } diff --git a/crates/cargo-test-support/containers/apache/Dockerfile b/crates/cargo-test-support/containers/apache/Dockerfile new file mode 100644 index 00000000000..87260241018 --- /dev/null +++ b/crates/cargo-test-support/containers/apache/Dockerfile @@ -0,0 +1,26 @@ +FROM httpd:2.4-alpine + +RUN apk add --no-cache git git-daemon openssl + +COPY bar /repos/bar +WORKDIR /repos/bar +RUN git config --global user.email "testuser@example.com" &&\ + git config --global user.name "Test User" &&\ + git init -b master . &&\ + git add Cargo.toml src &&\ + git commit -m "Initial commit" &&\ + mv .git ../bar.git &&\ + cd ../bar.git &&\ + git config --bool core.bare true &&\ + rm -rf ../bar +WORKDIR / + +EXPOSE 443 + +WORKDIR /usr/local/apache2/conf +COPY httpd-cargo.conf . +RUN cat httpd-cargo.conf >> httpd.conf +RUN openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \ + -keyout server.key -out server.crt \ + -subj "/emailAddress=webmaster@example.com/C=US/ST=California/L=San Francisco/O=Rust/OU=Cargo/CN=127.0.0.1" +WORKDIR / diff --git a/crates/cargo-test-support/containers/apache/bar/Cargo.toml b/crates/cargo-test-support/containers/apache/bar/Cargo.toml new file mode 100644 index 00000000000..84fd5d89b39 --- /dev/null +++ b/crates/cargo-test-support/containers/apache/bar/Cargo.toml @@ -0,0 +1,4 @@ +[package] +name = "bar" +version = "1.0.0" +edition = "2021" diff --git a/crates/cargo-test-support/containers/apache/bar/src/lib.rs b/crates/cargo-test-support/containers/apache/bar/src/lib.rs new file mode 100644 index 00000000000..ca74e3aec7e --- /dev/null +++ b/crates/cargo-test-support/containers/apache/bar/src/lib.rs @@ -0,0 +1 @@ +// Intentionally blank. diff --git a/crates/cargo-test-support/containers/apache/httpd-cargo.conf b/crates/cargo-test-support/containers/apache/httpd-cargo.conf new file mode 100644 index 00000000000..a4ba7d5240a --- /dev/null +++ b/crates/cargo-test-support/containers/apache/httpd-cargo.conf @@ -0,0 +1,12 @@ +SetEnv GIT_PROJECT_ROOT /repos +SetEnv GIT_HTTP_EXPORT_ALL +ScriptAlias /repos /usr/libexec/git-core/git-http-backend/ +LoadModule cgid_module modules/mod_cgid.so + + + Require all granted + + +Include conf/extra/httpd-ssl.conf +LoadModule ssl_module modules/mod_ssl.so +LoadModule socache_shmcb_module modules/mod_socache_shmcb.so diff --git a/crates/cargo-test-support/containers/sshd/Dockerfile b/crates/cargo-test-support/containers/sshd/Dockerfile new file mode 100644 index 00000000000..b52eefbad87 --- /dev/null +++ b/crates/cargo-test-support/containers/sshd/Dockerfile @@ -0,0 +1,29 @@ +FROM alpine:3.17 + +RUN apk add --no-cache openssh git +RUN ssh-keygen -A + +RUN addgroup -S testuser && adduser -S testuser -G testuser -s /bin/ash +# NOTE: Ideally the password should be set to *, but I am uncertain how to do +# that in alpine. It shouldn't matter since PermitEmptyPasswords is "no". +RUN passwd -u testuser + +RUN mkdir /repos && chown testuser /repos +COPY --chown=testuser:testuser bar /repos/bar +USER testuser +WORKDIR /repos/bar +RUN git config --global user.email "testuser@example.com" &&\ + git config --global user.name "Test User" &&\ + git init -b master . &&\ + git add Cargo.toml src &&\ + git commit -m "Initial commit" &&\ + mv .git ../bar.git &&\ + cd ../bar.git &&\ + git config --bool core.bare true &&\ + rm -rf ../bar +WORKDIR / +USER root + +EXPOSE 22 + +ENTRYPOINT ["/usr/sbin/sshd", "-D", "-E", "/var/log/auth.log"] diff --git a/crates/cargo-test-support/containers/sshd/bar/Cargo.toml b/crates/cargo-test-support/containers/sshd/bar/Cargo.toml new file mode 100644 index 00000000000..84fd5d89b39 --- /dev/null +++ b/crates/cargo-test-support/containers/sshd/bar/Cargo.toml @@ -0,0 +1,4 @@ +[package] +name = "bar" +version = "1.0.0" +edition = "2021" diff --git a/crates/cargo-test-support/containers/sshd/bar/src/lib.rs b/crates/cargo-test-support/containers/sshd/bar/src/lib.rs new file mode 100644 index 00000000000..ca74e3aec7e --- /dev/null +++ b/crates/cargo-test-support/containers/sshd/bar/src/lib.rs @@ -0,0 +1 @@ +// Intentionally blank. diff --git a/crates/cargo-test-support/src/containers.rs b/crates/cargo-test-support/src/containers.rs new file mode 100644 index 00000000000..17040d82abe --- /dev/null +++ b/crates/cargo-test-support/src/containers.rs @@ -0,0 +1,285 @@ +//! Support for testing using Docker containers. +//! +//! The [`Container`] type is a builder for configuring a container to run. +//! After you call `launch`, you can use the [`ContainerHandle`] to interact +//! with the running container. +//! +//! Tests using containers must use `#[cargo_test(container_test)]` to disable +//! them unless the CARGO_CONTAINER_TESTS environment variable is set. + +use cargo_util::ProcessBuilder; +use std::collections::HashMap; +use std::io::Read; +use std::path::PathBuf; +use std::process::Command; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Mutex; +use tar::Header; + +/// A builder for configuring a container to run. +pub struct Container { + /// The host directory that forms the basis of the Docker image. + build_context: PathBuf, + /// Files to copy over to the image. + files: Vec, +} + +/// A handle to a running container. +/// +/// You can use this to interact with the container. +pub struct ContainerHandle { + /// The name of the container. + name: String, + /// The IP address of the container. + /// + /// NOTE: This is currently unused, but may be useful so I left it in. + /// This can only be used on Linux. macOS and Windows docker doesn't allow + /// direct connection to the container. + pub ip_address: String, + /// Port mappings of container_port to host_port for ports exposed via EXPOSE. + pub port_mappings: HashMap, +} + +impl Container { + pub fn new(context_dir: &str) -> Container { + assert!(std::env::var_os("CARGO_CONTAINER_TESTS").is_some()); + let mut build_context = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + build_context.push("containers"); + build_context.push(context_dir); + Container { + build_context, + files: Vec::new(), + } + } + + /// Adds a file to be copied into the container. + pub fn file(mut self, file: MkFile) -> Self { + self.files.push(file); + self + } + + /// Starts the container. + pub fn launch(mut self) -> ContainerHandle { + static NEXT_ID: AtomicUsize = AtomicUsize::new(0); + + let id = NEXT_ID.fetch_add(1, Ordering::SeqCst); + let name = format!("cargo_test_{id}"); + remove_if_exists(&name); + self.create_container(&name); + self.copy_files(&name); + self.start_container(&name); + let info = self.container_inspect(&name); + let ip_address = if cfg!(target_os = "linux") { + info[0]["NetworkSettings"]["IPAddress"] + .as_str() + .unwrap() + .to_string() + } else { + // macOS and Windows can't make direct connections to the + // container. It only works through exposed ports or mapped ports. + "127.0.0.1".to_string() + }; + let port_mappings = self.port_mappings(&info); + self.wait_till_ready(&port_mappings); + + ContainerHandle { + name, + ip_address, + port_mappings, + } + } + + fn create_container(&self, name: &str) { + static BUILD_LOCK: Mutex<()> = Mutex::new(()); + + let image_base = self.build_context.file_name().unwrap(); + let image_name = format!("cargo-test-{}", image_base.to_str().unwrap()); + let _lock = BUILD_LOCK.lock().unwrap(); + ProcessBuilder::new("docker") + .args(&["build", "--tag", image_name.as_str()]) + .arg(&self.build_context) + .exec_with_output() + .unwrap(); + + ProcessBuilder::new("docker") + .args(&[ + "container", + "create", + "--publish-all", + "--rm", + "--name", + name, + ]) + .arg(image_name) + .exec_with_output() + .unwrap(); + } + + fn copy_files(&mut self, name: &str) { + if self.files.is_empty() { + return; + } + let mut ar = tar::Builder::new(Vec::new()); + let files = std::mem::replace(&mut self.files, Vec::new()); + for mut file in files { + ar.append_data(&mut file.header, &file.path, file.contents.as_slice()) + .unwrap(); + } + let ar = ar.into_inner().unwrap(); + ProcessBuilder::new("docker") + .args(&["cp", "-"]) + .arg(format!("{name}:/")) + .stdin(ar) + .exec_with_output() + .unwrap(); + } + + fn start_container(&self, name: &str) { + ProcessBuilder::new("docker") + .args(&["container", "start"]) + .arg(name) + .exec_with_output() + .unwrap(); + } + + fn container_inspect(&self, name: &str) -> serde_json::Value { + let output = ProcessBuilder::new("docker") + .args(&["inspect", name]) + .exec_with_output() + .unwrap(); + serde_json::from_slice(&output.stdout).unwrap() + } + + /// Returns the mapping of container_port->host_port for ports that were + /// exposed with EXPOSE. + fn port_mappings(&self, info: &serde_json::Value) -> HashMap { + info[0]["NetworkSettings"]["Ports"] + .as_object() + .unwrap() + .iter() + .map(|(key, value)| { + let key = key + .strip_suffix("/tcp") + .expect("expected TCP only ports") + .parse() + .unwrap(); + let values = value.as_array().unwrap(); + let value = values + .iter() + .find(|value| value["HostIp"].as_str().unwrap() == "0.0.0.0") + .expect("expected localhost IP"); + let host_port = value["HostPort"].as_str().unwrap().parse().unwrap(); + (key, host_port) + }) + .collect() + } + + fn wait_till_ready(&self, port_mappings: &HashMap) { + for port in port_mappings.values() { + let mut ok = false; + for _ in 0..30 { + match std::net::TcpStream::connect(format!("127.0.0.1:{port}")) { + Ok(_) => { + ok = true; + break; + } + Err(e) => { + if e.kind() != std::io::ErrorKind::ConnectionRefused { + panic!("unexpected localhost connection error: {e:?}"); + } + std::thread::sleep(std::time::Duration::new(1, 0)); + } + } + } + if !ok { + panic!("no listener on localhost port {port}"); + } + } + } +} + +impl ContainerHandle { + /// Executes a program inside a running container. + pub fn exec(&self, args: &[&str]) -> std::process::Output { + ProcessBuilder::new("docker") + .args(&["container", "exec", &self.name]) + .args(args) + .exec_with_output() + .unwrap() + } + + /// Returns the contents of a file inside the container. + pub fn read_file(&self, path: &str) -> String { + let output = ProcessBuilder::new("docker") + .args(&["cp", &format!("{}:{}", self.name, path), "-"]) + .exec_with_output() + .unwrap(); + let mut ar = tar::Archive::new(output.stdout.as_slice()); + let mut entry = ar.entries().unwrap().next().unwrap().unwrap(); + let mut contents = String::new(); + entry.read_to_string(&mut contents).unwrap(); + contents + } +} + +impl Drop for ContainerHandle { + fn drop(&mut self) { + // To help with debugging, this will keep the container alive. + if std::env::var_os("CARGO_CONTAINER_TEST_KEEP").is_some() { + return; + } + remove_if_exists(&self.name); + } +} + +fn remove_if_exists(name: &str) { + if let Err(e) = Command::new("docker") + .args(&["container", "rm", "--force", name]) + .output() + { + panic!("failed to run docker: {e}"); + } +} + +/// Builder for configuring a file to copy into a container. +pub struct MkFile { + path: String, + contents: Vec, + header: Header, +} + +impl MkFile { + /// Defines a file to add to the container. + /// + /// This should be passed to `Container::file`. + /// + /// The path is the path inside the container to create the file. + pub fn path(path: &str) -> MkFile { + MkFile { + path: path.to_string(), + contents: Vec::new(), + header: Header::new_gnu(), + } + } + + pub fn contents(mut self, contents: impl Into>) -> Self { + self.contents = contents.into(); + self.header.set_size(self.contents.len() as u64); + self + } + + pub fn mode(mut self, mode: u32) -> Self { + self.header.set_mode(mode); + self + } + + pub fn uid(mut self, uid: u64) -> Self { + self.header.set_uid(uid); + self + } + + pub fn gid(mut self, gid: u64) -> Self { + self.header.set_gid(gid); + self + } +} diff --git a/crates/cargo-test-support/src/lib.rs b/crates/cargo-test-support/src/lib.rs index 4ce0162bfbd..1ca5f525cf3 100644 --- a/crates/cargo-test-support/src/lib.rs +++ b/crates/cargo-test-support/src/lib.rs @@ -70,6 +70,7 @@ pub fn panic_error(what: &str, err: impl Into) -> ! { pub use cargo_test_macro::cargo_test; pub mod compare; +pub mod containers; pub mod cross_compile; mod diff; pub mod git; @@ -1227,6 +1228,8 @@ pub trait TestEnv: Sized { // should hopefully not surprise us as we add cargo features over time and // cargo rides the trains. .env("__CARGO_TEST_CHANNEL_OVERRIDE_DO_NOT_USE_THIS", "stable") + // Keeps cargo within its sandbox. + .env("__CARGO_TEST_DISABLE_GLOBAL_KNOWN_HOST", "1") // For now disable incremental by default as support hasn't ridden to the // stable channel yet. Once incremental support hits the stable compiler we // can switch this to one and then fix the tests. @@ -1247,11 +1250,15 @@ pub trait TestEnv: Sized { .env_remove("GIT_AUTHOR_EMAIL") .env_remove("GIT_COMMITTER_NAME") .env_remove("GIT_COMMITTER_EMAIL") + .env_remove("SSH_AUTH_SOCK") // ensure an outer agent is never contacted .env_remove("MSYSTEM"); // assume cmd.exe everywhere on windows if cfg!(target_os = "macos") { // Work-around a bug in macOS 10.15, see `link_or_copy` for details. self = self.env("__CARGO_COPY_DONT_LINK_DO_NOT_USE_THIS", "1"); } + if cfg!(windows) { + self = self.env("USERPROFILE", paths::home()); + } self } diff --git a/src/cargo/sources/git/known_hosts.rs b/src/cargo/sources/git/known_hosts.rs index ca732fafd4a..c8466d6079c 100644 --- a/src/cargo/sources/git/known_hosts.rs +++ b/src/cargo/sources/git/known_hosts.rs @@ -358,7 +358,8 @@ fn check_ssh_known_hosts_loaded( /// Returns a list of files to try loading OpenSSH-formatted known hosts. fn known_host_files() -> Vec { let mut result = Vec::new(); - if cfg!(unix) { + if std::env::var_os("__CARGO_TEST_DISABLE_GLOBAL_KNOWN_HOST").is_some() { + } else if cfg!(unix) { result.push(PathBuf::from("/etc/ssh/ssh_known_hosts")); } else if cfg!(windows) { // The msys/cygwin version of OpenSSH uses `/etc` from the posix root diff --git a/src/cargo/util/network.rs b/src/cargo/util/network.rs index cc3ef980fe5..337eef0b950 100644 --- a/src/cargo/util/network.rs +++ b/src/cargo/util/network.rs @@ -55,7 +55,7 @@ fn maybe_spurious(err: &Error) -> bool { git2::ErrorClass::Net | git2::ErrorClass::Os | git2::ErrorClass::Zlib - | git2::ErrorClass::Http => return true, + | git2::ErrorClass::Http => return git_err.code() != git2::ErrorCode::Certificate, _ => (), } } diff --git a/src/doc/contrib/src/tests/running.md b/src/doc/contrib/src/tests/running.md index b2c4659b4fe..dc306fbb4b2 100644 --- a/src/doc/contrib/src/tests/running.md +++ b/src/doc/contrib/src/tests/running.md @@ -39,3 +39,30 @@ the `CARGO_RUN_BUILD_STD_TESTS=1` environment variable and running `cargo test --test build-std`. This requires the nightly channel, and also requires the `rust-src` component installed with `rustup component add rust-src --toolchain=nightly`. + +## Running public network tests + +Some (very rare) tests involve connecting to the public internet. +These tests are disabled by default, +but you can run them by setting the `CARGO_PUBLIC_NETWORK_TESTS=1` environment variable. +Additionally our CI suite has a smoke test for fetching dependencies. +For most contributors, you will never need to bother with this. + +## Running container tests + +Tests marked with `container_test` involve running Docker to test more complex configurations. +These tests are disabled by default, +but you can run them by setting the `CARGO_CONTAINER_TESTS=1` environment variable. +You will need to have Docker installed and running to use these. + +> Note: Container tests mostly do not work on Windows. +> * The SSH tests require ssh-agent, but the two versions of ssh-agent +> on Windows are not suitable for testing. +> * The Microsoft version of ssh-agent runs as a global service, and can't be isolated per test. +> * The mingw/cygwin one can't be accessed from a Windows executable like cargo. +> * Pageant similarly does not seem to have a way to isolate it (and I'm not certain it can be driven completely from the command-line). +> +> The tests also can't run on Windows CI because the Docker that is preinstalled doesn't support Linux containers, and setting up Windows containers is a pain. +> +> macOS should work with Docker installed and running, +> but unfortunately the tests are not run on CI because Docker is not available. diff --git a/src/doc/contrib/src/tests/writing.md b/src/doc/contrib/src/tests/writing.md index 886429e58c0..0077e8f2d34 100644 --- a/src/doc/contrib/src/tests/writing.md +++ b/src/doc/contrib/src/tests/writing.md @@ -92,6 +92,19 @@ The options it supports are: * `>=1.64` — This indicates that the test will only run with the given version of `rustc` or newer. This can be used when a new `rustc` feature has been stabilized that the test depends on. If this is specified, a `reason` is required to explain why it is being checked. +* `public_network_test` — This tests contacts the public internet. + These tests are disabled unless the `CARGO_PUBLIC_NETWORK_TESTS` environment variable is set. + Use of this should be *extremely rare*, please avoid using it if possible. + The hosts it contacts should have a relatively high confidence that they are reliable and stable (such as github.com), especially in CI. + The tests should be carefully considered for developer security and privacy as well. +* `container_test` — This indicates that it is a test that uses Docker. + These tests are disabled unless the `CARGO_CONTAINER_TESTS` environment variable is set. + This requires that you have Docker installed. + The SSH tests also assume that you have OpenSSH installed. + These should work on Linux, macOS, and Windows where possible. + Unfortunately these tests are not run in CI for macOS or Windows (no Docker on macOS, and Windows does not support Linux images). + See [`crates/cargo-test-support/src/containers.rs`](https://github.com/rust-lang/cargo/blob/master/crates/cargo-test-support/src/containers.rs) for more on writing these tests. + #### Testing Nightly Features diff --git a/tests/testsuite/https.rs b/tests/testsuite/https.rs new file mode 100644 index 00000000000..c7aec911105 --- /dev/null +++ b/tests/testsuite/https.rs @@ -0,0 +1,152 @@ +//! Network tests for https transport. +//! +//! Note that these tests will generally require setting CARGO_CONTAINER_TESTS +//! or CARGO_PUBLIC_NETWORK_TESTS. + +use cargo_test_support::containers::Container; +use cargo_test_support::project; + +#[cargo_test(container_test)] +fn self_signed_should_fail() { + // Cargo should not allow a connection to a self-signed certificate. + let apache = Container::new("apache").launch(); + let port = apache.port_mappings[&443]; + let url = format!("https://127.0.0.1:{port}/repos/bar.git"); + let p = project() + .file( + "Cargo.toml", + &format!( + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bar = {{ git = "{url}" }} + "# + ), + ) + .file("src/lib.rs", "") + .build(); + // I think the text here depends on the curl backend. + let err_msg = if cfg!(target_os = "macos") { + "untrusted connection error; class=Ssl (16); code=Certificate (-17)" + } else if cfg!(unix) { + "the SSL certificate is invalid; class=Ssl (16); code=Certificate (-17)" + } else if cfg!(windows) { + "user cancelled certificate check; class=Http (34); code=Certificate (-17)" + } else { + panic!("target not supported"); + }; + p.cargo("fetch") + .with_status(101) + .with_stderr(&format!( + "\ +[UPDATING] git repository `https://127.0.0.1:[..]/repos/bar.git` +error: failed to get `bar` as a dependency of package `foo v0.1.0 ([ROOT]/foo)` + +Caused by: + failed to load source for dependency `bar` + +Caused by: + Unable to update https://127.0.0.1:[..]/repos/bar.git + +Caused by: + failed to clone into: [ROOT]/home/.cargo/git/db/bar-[..] + +Caused by: + network failure seems to have happened + if a proxy or similar is necessary `net.git-fetch-with-cli` may help here + https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli + +Caused by: + {err_msg} +" + )) + .run(); +} + +#[cargo_test(container_test)] +fn self_signed_with_cacert() { + // When using cainfo, that should allow a connection to a self-signed cert. + + if cfg!(target_os = "macos") { + // This test only seems to work with the + // curl-sys/force-system-lib-on-osx feature enabled. For some reason + // SecureTransport doesn't seem to like the self-signed certificate. + // It works if the certificate is manually approved via Keychain + // Access. The system libcurl is built with a LibreSSL fallback which + // is used when CAINFO is set, which seems to work correctly. This + // could use some more investigation. The official Rust binaries use + // curl-sys/force-system-lib-on-osx so it is mostly an issue for local + // testing. + // + // The error is: + // [60] SSL peer certificate or SSH remote key was not OK (SSL: + // certificate verification failed (result: 5)); class=Net (12) + let curl_v = curl::Version::get(); + if curl_v.vendored() { + eprintln!( + "vendored curl not supported on macOS, \ + set curl-sys/force-system-lib-on-osx to enable" + ); + return; + } + } + + let apache = Container::new("apache").launch(); + let port = apache.port_mappings[&443]; + let url = format!("https://127.0.0.1:{port}/repos/bar.git"); + let server_crt = apache.read_file("/usr/local/apache2/conf/server.crt"); + let p = project() + .file( + "Cargo.toml", + &format!( + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bar = {{ git = "{url}" }} + "# + ), + ) + .file("src/lib.rs", "") + .file( + ".cargo/config.toml", + &format!( + r#" + [http] + cainfo = "server.crt" + "# + ), + ) + .file("server.crt", &server_crt) + .build(); + p.cargo("fetch") + .with_stderr("[UPDATING] git repository `https://127.0.0.1:[..]/repos/bar.git`") + .run(); +} + +#[cargo_test(public_network_test)] +fn github_works() { + // Check that an https connection to github.com works. + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bitflags = { git = "https://github.com/rust-lang/bitflags.git", tag="1.3.2" } + "#, + ) + .file("src/lib.rs", "") + .build(); + p.cargo("fetch") + .with_stderr("[UPDATING] git repository `https://github.com/rust-lang/bitflags.git`") + .run(); +} diff --git a/tests/testsuite/main.rs b/tests/testsuite/main.rs index b71a9a95ece..76d3f22dae9 100644 --- a/tests/testsuite/main.rs +++ b/tests/testsuite/main.rs @@ -61,6 +61,7 @@ mod git_auth; mod git_gc; mod glob_targets; mod help; +mod https; mod inheritable_workspace_fields; mod init; mod install; @@ -121,6 +122,7 @@ mod rustflags; mod search; mod shell_quoting; mod source_replacement; +mod ssh; mod standard_lib; mod test; mod timings; diff --git a/tests/testsuite/ssh.rs b/tests/testsuite/ssh.rs new file mode 100644 index 00000000000..b0fe9ffff63 --- /dev/null +++ b/tests/testsuite/ssh.rs @@ -0,0 +1,548 @@ +//! Network tests for SSH connections. +//! +//! Note that these tests will generally require setting CARGO_CONTAINER_TESTS +//! or CARGO_PUBLIC_NETWORK_TESTS. +//! +//! NOTE: The container tests almost certainly won't work on Windows. + +use cargo_test_support::containers::{Container, ContainerHandle, MkFile}; +use cargo_test_support::{paths, process, project, Project}; +use std::fs; +use std::io::Write; +use std::path::PathBuf; + +fn ssh_repo_url(container: &ContainerHandle, name: &str) -> String { + let port = container.port_mappings[&22]; + format!("ssh://testuser@127.0.0.1:{port}/repos/{name}.git") +} + +/// The path to the client's private key. +fn key_path() -> PathBuf { + paths::home().join(".ssh/id_ed25519") +} + +/// Generates the SSH keys for authenticating into the container. +fn gen_ssh_keys() -> String { + let path = key_path(); + process("ssh-keygen") + .args(&["-t", "ed25519", "-N", "", "-f"]) + .arg(&path) + .exec_with_output() + .unwrap(); + let pub_key = path.with_extension("pub"); + fs::read_to_string(pub_key).unwrap() +} + +/// Handler for running ssh-agent for SSH authentication. +/// +/// Be sure to set `SSH_AUTH_SOCK` when running a process in order to use the +/// agent. Keys will need to be copied into the container with the +/// `authorized_keys()` method. +struct Agent { + sock: PathBuf, + pid: String, + ssh_dir: PathBuf, + pub_key: String, +} + +impl Agent { + fn launch() -> Agent { + let ssh_dir = paths::home().join(".ssh"); + fs::create_dir(&ssh_dir).unwrap(); + let pub_key = gen_ssh_keys(); + + let sock = paths::root().join("agent"); + let output = process("ssh-agent") + .args(&["-s", "-a"]) + .arg(&sock) + .exec_with_output() + .unwrap(); + let stdout = std::str::from_utf8(&output.stdout).unwrap(); + let start = stdout.find("SSH_AGENT_PID=").unwrap() + 14; + let end = &stdout[start..].find(';').unwrap(); + let pid = (&stdout[start..start + end]).to_string(); + eprintln!("SSH_AGENT_PID={pid}"); + process("ssh-add") + .arg(key_path()) + .env("SSH_AUTH_SOCK", &sock) + .exec_with_output() + .unwrap(); + Agent { + sock, + pid, + ssh_dir, + pub_key, + } + } + + /// Returns a `MkFile` which can be passed into the `Container` builder to + /// copy an `authorized_keys` file containing this agent's public key. + fn authorized_keys(&self) -> MkFile { + MkFile::path("home/testuser/.ssh/authorized_keys") + .contents(self.pub_key.as_bytes()) + .mode(0o600) + .uid(100) + .gid(101) + } +} + +impl Drop for Agent { + fn drop(&mut self) { + if let Err(e) = process("ssh-agent") + .args(&["-k", "-a"]) + .arg(&self.sock) + .env("SSH_AGENT_PID", &self.pid) + .exec_with_output() + { + eprintln!("failed to stop ssh-agent: {e:?}"); + } + } +} + +/// Common project used for several tests. +fn foo_bar_project(url: &str) -> Project { + project() + .file( + "Cargo.toml", + &format!( + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bar = {{ git = "{url}" }} + "# + ), + ) + .file("src/lib.rs", "") + .build() +} + +#[cargo_test(container_test)] +fn no_known_host() { + // When host is not known, it should show an error. + let sshd = Container::new("sshd").launch(); + let url = ssh_repo_url(&sshd, "bar"); + let p = foo_bar_project(&url); + p.cargo("fetch") + .with_status(101) + .with_stderr( + "\ +[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git` +error: failed to get `bar` as a dependency of package `foo v0.1.0 ([ROOT]/foo)` + +Caused by: + failed to load source for dependency `bar` + +Caused by: + Unable to update ssh://testuser@127.0.0.1:[..]/repos/bar.git + +Caused by: + failed to clone into: [ROOT]/home/.cargo/git/db/bar-[..] + +Caused by: + error: unknown SSH host key + The SSH host key for `[127.0.0.1]:[..]` is not known and cannot be validated. + + To resolve this issue, add the host key to the `net.ssh.known-hosts` array in \ + your Cargo configuration (such as [ROOT]/home/.cargo/config.toml) or in your \ + OpenSSH known_hosts file at [ROOT]/home/.ssh/known_hosts + + The key to add is: + + [127.0.0.1]:[..] ecdsa-sha2-nistp256 AAAA[..] + + The ECDSA key fingerprint is: SHA256:[..] + This fingerprint should be validated with the server administrator that it is correct. + + See https://doc.rust-lang.org/stable/cargo/appendix/git-authentication.html#ssh-known-hosts \ + for more information. +", + ) + .run(); +} + +#[cargo_test(container_test)] +fn known_host_works() { + // The key displayed in the error message should work when added to known_hosts. + let agent = Agent::launch(); + let sshd = Container::new("sshd") + .file(agent.authorized_keys()) + .launch(); + let url = ssh_repo_url(&sshd, "bar"); + let p = foo_bar_project(&url); + let output = p + .cargo("fetch") + .env("SSH_AUTH_SOCK", &agent.sock) + .build_command() + .output() + .unwrap(); + let stderr = std::str::from_utf8(&output.stderr).unwrap(); + + // Validate the fingerprint while we're here. + let fingerprint = stderr + .lines() + .find(|line| line.starts_with(" The ECDSA key fingerprint")) + .unwrap() + .trim(); + let fingerprint = &fingerprint[30..]; + let finger_out = sshd.exec(&["ssh-keygen", "-l", "-f", "/etc/ssh/ssh_host_ecdsa_key.pub"]); + let gen_finger = std::str::from_utf8(&finger_out.stdout).unwrap(); + // + let gen_finger = gen_finger.split_whitespace().nth(1).unwrap(); + assert_eq!(fingerprint, gen_finger); + + // Add the key to known_hosts, and try again. + let key = stderr + .lines() + .find(|line| line.starts_with(" [127.0.0.1]:")) + .unwrap() + .trim(); + fs::write(agent.ssh_dir.join("known_hosts"), key).unwrap(); + p.cargo("fetch") + .env("SSH_AUTH_SOCK", &agent.sock) + .with_stderr("[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git`") + .run(); +} + +#[cargo_test(container_test)] +fn same_key_different_hostname() { + // The error message should mention if an identical key was found. + let agent = Agent::launch(); + let sshd = Container::new("sshd").launch(); + + let hostkey = sshd.read_file("/etc/ssh/ssh_host_ecdsa_key.pub"); + let known_hosts = format!("example.com {hostkey}"); + fs::write(agent.ssh_dir.join("known_hosts"), known_hosts).unwrap(); + + let url = ssh_repo_url(&sshd, "bar"); + let p = foo_bar_project(&url); + p.cargo("fetch") + .with_status(101) + .with_stderr( + "\ +[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git` +error: failed to get `bar` as a dependency of package `foo v0.1.0 ([ROOT]/foo)` + +Caused by: + failed to load source for dependency `bar` + +Caused by: + Unable to update ssh://testuser@127.0.0.1:[..]/repos/bar.git + +Caused by: + failed to clone into: [ROOT]/home/.cargo/git/db/bar-[..] + +Caused by: + error: unknown SSH host key + The SSH host key for `[127.0.0.1]:[..]` is not known and cannot be validated. + + To resolve this issue, add the host key to the `net.ssh.known-hosts` array in \ + your Cargo configuration (such as [ROOT]/home/.cargo/config.toml) or in your \ + OpenSSH known_hosts file at [ROOT]/home/.ssh/known_hosts + + The key to add is: + + [127.0.0.1]:[..] ecdsa-sha2-nistp256 AAAA[..] + + The ECDSA key fingerprint is: SHA256:[..] + This fingerprint should be validated with the server administrator that it is correct. + Note: This host key was found, but is associated with a different host: + [ROOT]/home/.ssh/known_hosts line 1: example.com + + See https://doc.rust-lang.org/stable/cargo/appendix/git-authentication.html#ssh-known-hosts \ + for more information. +", + ) + .run(); +} + +#[cargo_test(container_test)] +fn known_host_without_port() { + // A known_host entry without a port should match a connection to a non-standard port. + let agent = Agent::launch(); + let sshd = Container::new("sshd") + .file(agent.authorized_keys()) + .launch(); + + let hostkey = sshd.read_file("/etc/ssh/ssh_host_ecdsa_key.pub"); + // The important part of this test is that this line does not have a port. + let known_hosts = format!("127.0.0.1 {hostkey}"); + fs::write(agent.ssh_dir.join("known_hosts"), known_hosts).unwrap(); + let url = ssh_repo_url(&sshd, "bar"); + let p = foo_bar_project(&url); + p.cargo("fetch") + .env("SSH_AUTH_SOCK", &agent.sock) + .with_stderr("[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git`") + .run(); +} + +#[cargo_test(container_test)] +fn hostname_case_insensitive() { + // hostname checking should be case-insensitive. + let agent = Agent::launch(); + let sshd = Container::new("sshd") + .file(agent.authorized_keys()) + .launch(); + + // Consider using `gethostname-rs` instead? + let hostname = process("hostname").exec_with_output().unwrap(); + let hostname = std::str::from_utf8(&hostname.stdout).unwrap().trim(); + let inv_hostname = if hostname.chars().any(|c| c.is_lowercase()) { + hostname.to_uppercase() + } else { + // There should be *some* chars in the name. + assert!(hostname.chars().any(|c| c.is_uppercase())); + hostname.to_lowercase() + }; + eprintln!("converted {hostname} to {inv_hostname}"); + + let hostkey = sshd.read_file("/etc/ssh/ssh_host_ecdsa_key.pub"); + let known_hosts = format!("{inv_hostname} {hostkey}"); + fs::write(agent.ssh_dir.join("known_hosts"), known_hosts).unwrap(); + let port = sshd.port_mappings[&22]; + let url = format!("ssh://testuser@{hostname}:{port}/repos/bar.git"); + let p = foo_bar_project(&url); + p.cargo("fetch") + .env("SSH_AUTH_SOCK", &agent.sock) + .with_stderr(&format!( + "[UPDATING] git repository `ssh://testuser@{hostname}:{port}/repos/bar.git`" + )) + .run(); +} + +#[cargo_test(container_test)] +fn invalid_key_error() { + // An error when a known_host value doesn't match. + let agent = Agent::launch(); + let sshd = Container::new("sshd") + .file(agent.authorized_keys()) + .launch(); + + let port = sshd.port_mappings[&22]; + let known_hosts = format!( + "[127.0.0.1]:{port} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLqLMclVr7MDuaVsm3sEnnq2OrGxTFiHSw90wd6N14BU8xVC9cZldC3rJ58Wmw6bEVKPjk7foNG0lHwS5bCKX+U=\n" + ); + fs::write(agent.ssh_dir.join("known_hosts"), known_hosts).unwrap(); + let url = ssh_repo_url(&sshd, "bar"); + let p = foo_bar_project(&url); + p.cargo("fetch") + .env("SSH_AUTH_SOCK", &agent.sock) + .with_status(101) + .with_stderr(&format!("\ +[UPDATING] git repository `ssh://testuser@127.0.0.1:{port}/repos/bar.git` +error: failed to get `bar` as a dependency of package `foo v0.1.0 ([ROOT]/foo)` + +Caused by: + failed to load source for dependency `bar` + +Caused by: + Unable to update ssh://testuser@127.0.0.1:{port}/repos/bar.git + +Caused by: + failed to clone into: [ROOT]/home/.cargo/git/db/bar-[..] + +Caused by: + error: SSH host key has changed for `[127.0.0.1]:{port}` + ********************************* + * WARNING: HOST KEY HAS CHANGED * + ********************************* + This may be caused by a man-in-the-middle attack, or the server may have changed its host key. + + The ECDSA fingerprint for the key from the remote host is: + SHA256:[..] + + You are strongly encouraged to contact the server administrator for `[127.0.0.1]:{port}` \ + to verify that this new key is correct. + + If you can verify that the server has a new key, you can resolve this error by \ + removing the old ecdsa-sha2-nistp256 key for `[127.0.0.1]:{port}` located at \ + [ROOT]/home/.ssh/known_hosts line 1, and adding the new key to the \ + `net.ssh.known-hosts` array in your Cargo configuration (such as \ + [ROOT]/home/.cargo/config.toml) or in your OpenSSH known_hosts file at \ + [ROOT]/home/.ssh/known_hosts + + The key provided by the remote host is: + + [127.0.0.1]:{port} ecdsa-sha2-nistp256 [..] + + See https://doc.rust-lang.org/stable/cargo/appendix/git-authentication.html#ssh-known-hosts for more information. +")) + .run(); + // Add the key, it should work even with the old key left behind. + let hostkey = sshd.read_file("/etc/ssh/ssh_host_ecdsa_key.pub"); + let known_hosts_path = agent.ssh_dir.join("known_hosts"); + let mut f = fs::OpenOptions::new() + .append(true) + .open(known_hosts_path) + .unwrap(); + write!(f, "[127.0.0.1]:{port} {hostkey}").unwrap(); + drop(f); + p.cargo("fetch") + .env("SSH_AUTH_SOCK", &agent.sock) + .with_stderr("[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git`") + .run(); +} + +#[cargo_test(public_network_test)] +fn invalid_github_key() { + // A key for github.com in known_hosts should override the built-in key. + // This uses a bogus key which should result in an error. + let ssh_dir = paths::home().join(".ssh"); + fs::create_dir(&ssh_dir).unwrap(); + let known_hosts = "\ + github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLqLMclVr7MDuaVsm3sEnnq2OrGxTFiHSw90wd6N14BU8xVC9cZldC3rJ58Wmw6bEVKPjk7foNG0lHwS5bCKX+U=\n\ + github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgi+8rMcyFCBq5y7BXrb2aaYGhMjlU3QDy7YDvtNL5KSecYOsaqQHaXr87Bbx0EEkgbhK4kVMkmThlCoNITQS9Vc3zIMQ+Tg6+O4qXx719uCzywl50Tb5tDqPGMj54jcq3VUiu/dvse0yeehyvzoPNWewgGWLx11KI4A4wOwMnc6guhculEWe9DjGEjUQ34lPbmdfu/Hza7ZVu/RhgF/wc43uzXWB2KpMEqtuY1SgRlCZqTASoEtfKZi0AuM7AEdOwE5aTotS4CQZHWimb1bMFpF4DAq92CZ8Jhrm4rWETbO29WmjviCJEA3KNQyd3oA7H9AE9z/22PJaVEmjiZZ+wyLgwyIpOlsnHYNEdGeQMQ4SgLRkARLwcnKmByv1AAxsBW4LI3Os4FpwxVPdXHcBebydtvxIsbtUVkkq99nbsIlnSRFSTvb0alrdzRuKTdWpHtN1v9hagFqmeCx/kJfH76NXYBbtaWZhSOnxfEbhLYuOb+IS4jYzHAIkzy9FjVuk=\n\ + ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEeMB6BUAW6FfvfLxRO3kGASe0yXnrRT4kpqncsup2b2\n"; + fs::write(ssh_dir.join("known_hosts"), known_hosts).unwrap(); + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bitflags = { git = "ssh://git@github.com/rust-lang/bitflags.git", tag = "1.3.2" } + "#, + ) + .file("src/lib.rs", "") + .build(); + p.cargo("fetch") + .with_status(101) + .with_stderr_contains(" error: SSH host key has changed for `github.com`") + .run(); +} + +#[cargo_test(public_network_test)] +fn bundled_github_works() { + // The bundled key for github.com works. + // + // Use a bogus auth sock to force an authentication error. + // On Windows, if the agent service is running, it could allow a + // successful authentication. + // + // If the bundled hostkey did not work, it would result in an "unknown SSH + // host key" instead. + let bogus_auth_sock = paths::home().join("ssh_auth_sock"); + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bitflags = { git = "ssh://git@github.com/rust-lang/bitflags.git", tag = "1.3.2" } + "#, + ) + .file("src/lib.rs", "") + .build(); + let err = if cfg!(windows) { + "error authenticating: unable to connect to agent pipe; class=Ssh (23)" + } else { + "error authenticating: failed connecting with agent; class=Ssh (23)" + }; + p.cargo("fetch") + .env("SSH_AUTH_SOCK", &bogus_auth_sock) + .with_status(101) + .with_stderr(&format!( + "\ +[UPDATING] git repository `ssh://git@github.com/rust-lang/bitflags.git` +error: failed to get `bitflags` as a dependency of package `foo v0.1.0 ([ROOT]/foo)` + +Caused by: + failed to load source for dependency `bitflags` + +Caused by: + Unable to update ssh://git@github.com/rust-lang/bitflags.git?tag=1.3.2 + +Caused by: + failed to clone into: [ROOT]/home/.cargo/git/db/bitflags-[..] + +Caused by: + failed to authenticate when downloading repository + + * attempted ssh-agent authentication, but no usernames succeeded: `git` + + if the git CLI succeeds then `net.git-fetch-with-cli` may help here + https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli + +Caused by: + {err} +" + )) + .run(); + + // Explicit :22 should also work with bundled. + p.change_file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bitflags = { git = "ssh://git@github.com:22/rust-lang/bitflags.git", tag = "1.3.2" } + "#, + ); + p.cargo("fetch") + .env("SSH_AUTH_SOCK", &bogus_auth_sock) + .with_status(101) + .with_stderr(&format!( + "\ +[UPDATING] git repository `ssh://git@github.com:22/rust-lang/bitflags.git` +error: failed to get `bitflags` as a dependency of package `foo v0.1.0 ([ROOT]/foo)` + +Caused by: + failed to load source for dependency `bitflags` + +Caused by: + Unable to update ssh://git@github.com:22/rust-lang/bitflags.git?tag=1.3.2 + +Caused by: + failed to clone into: [ROOT]/home/.cargo/git/db/bitflags-[..] + +Caused by: + failed to authenticate when downloading repository + + * attempted ssh-agent authentication, but no usernames succeeded: `git` + + if the git CLI succeeds then `net.git-fetch-with-cli` may help here + https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli + +Caused by: + {err} +" + )) + .run(); +} + +#[cargo_test(container_test)] +fn ssh_key_in_config() { + // known_host in config works. + let agent = Agent::launch(); + let sshd = Container::new("sshd") + .file(agent.authorized_keys()) + .launch(); + let hostkey = sshd.read_file("/etc/ssh/ssh_host_ecdsa_key.pub"); + let url = ssh_repo_url(&sshd, "bar"); + let p = foo_bar_project(&url); + p.change_file( + ".cargo/config.toml", + &format!( + r#" + [net.ssh] + known-hosts = ['127.0.0.1 {}'] + "#, + hostkey.trim() + ), + ); + p.cargo("fetch") + .env("SSH_AUTH_SOCK", &agent.sock) + .with_stderr("[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git`") + .run(); +}