Merge branch 'main' into audit

.github/workflows/ci_rust.yml

@@ -58,6 +58,10 @@ jobs:
         working-directory: ${{env.ROOT_PATH}}
         run: cargo test --features unstable-renegotiate
 
+      - name: Network-enabled integration tests
+        working-directory: ${{env.ROOT_PATH}}/integration
+        run: RUST_LOG=TRACE cargo test --features network-tests
+
       - name: Test external build
         # if this test is failing, make sure that api headers are appropriately
         # included. For a symbol to be visible in a shared lib, the

bindings/rust/integration/Cargo.toml

@@ -5,7 +5,34 @@ authors = ["AWS s2n"]
 edition = "2021"
 publish = false
 
+[features]
+default = ["pq"]
+
+# Network tests are useful but relatively slow and inherently flaky. So they are
+# behind this feature flag.
+network-tests = []
+
+# Not all libcryptos support PQ capabilities. Tests relying on PQ functionality
+# can be disabled by turning off this feature.
+pq = []
+
 [dependencies]
-s2n-tls = { path = "../s2n-tls"}
+s2n-tls = { path = "../s2n-tls", features = ["unstable-testing"]}
+s2n-tls-hyper = { path = "../s2n-tls-hyper" }
+s2n-tls-tokio = { path = "../s2n-tls-tokio" }
 s2n-tls-sys = { path = "../s2n-tls-sys" }
 
+[dev-dependencies]
+tokio = { version = "1", features = ["macros", "test-util"] }
+
+tracing = "0.1"
+tracing-subscriber = "0.3"
+# TODO: Unpin when s2n-tls MSRV >= 1.71, https://github.com/aws/s2n-tls/issues/4893
+test-log = { version = "=0.2.14", default-features = false, features = ["trace"]}
+test-log-macros = "=0.2.14"
+
+http = "1.1"
+http-body-util = "0.1"
+bytes = "1.8"
+hyper = "1.5"
+hyper-util = "0.1"

bindings/rust/integration/src/lib.rs

@@ -0,0 +1,31 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#[cfg(all(feature = "network-tests", test))]
+mod network;
+
+#[cfg(test)]
+mod tests {
+    use s2n_tls::{
+        security::Policy,
+        testing::{self, TestPair},
+    };
+
+    /// This test provides a helpful debug message if the PQ feature is incorrectly
+    /// configured.
+    #[cfg(feature = "pq")]
+    #[test]
+    fn pq_sanity_check() -> Result<(), Box<dyn std::error::Error>> {
+        let config = testing::build_config(&Policy::from_version("KMS-PQ-TLS-1-0-2020-07")?)?;
+        let mut pair = TestPair::from_config(&config);
+        pair.handshake()?;
+
+        if pair.client.kem_name().is_none() {
+            panic!(
+                "PQ tests are enabled, but PQ functionality is unavailable. \
+                Are you sure that the libcrypto supports PQ?"
+            );
+        }
+        Ok(())
+    }
+}

bindings/rust/integration/src/network/https_client.rs

@@ -0,0 +1,97 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use bytes::Bytes;
+use http::{Response, StatusCode, Uri};
+use http_body_util::{BodyExt, Empty};
+use hyper::body::Incoming;
+use hyper_util::{client::legacy::Client, rt::TokioExecutor};
+use s2n_tls::{
+    config::Config,
+    security::{self, Policy},
+};
+use s2n_tls_hyper::connector::HttpsConnector;
+use std::str::FromStr;
+
+#[derive(Debug)]
+struct TestCase {
+    pub query_target: &'static str,
+    pub expected_status_code: u16,
+}
+
+impl TestCase {
+    const fn new(domain: &'static str, expected_status_code: u16) -> Self {
+        TestCase {
+            query_target: domain,
+            expected_status_code,
+        }
+    }
+}
+
+const TEST_CASES: &[TestCase] = &[
+    // this is a link to the s2n-tls unit test coverage report, hosted on cloudfront
+    TestCase::new("https://dx1inn44oyl7n.cloudfront.net/main/index.html", 200),
+    // this is a link to a non-existent S3 item
+    TestCase::new("https://notmybucket.s3.amazonaws.com/folder/afile.jpg", 403),
+    TestCase::new("https://www.amazon.com", 200),
+    TestCase::new("https://www.apple.com", 200),
+    TestCase::new("https://www.att.com", 200),
+    TestCase::new("https://www.cloudflare.com", 200),
+    TestCase::new("https://www.ebay.com", 200),
+    TestCase::new("https://www.google.com", 200),
+    TestCase::new("https://www.mozilla.org", 200),
+    TestCase::new("https://www.netflix.com", 200),
+    TestCase::new("https://www.openssl.org", 200),
+    TestCase::new("https://www.t-mobile.com", 200),
+    TestCase::new("https://www.verizon.com", 200),
+    TestCase::new("https://www.wikipedia.org", 200),
+    TestCase::new("https://www.yahoo.com", 200),
+    TestCase::new("https://www.youtube.com", 200),
+    TestCase::new("https://www.github.com", 301),
+    TestCase::new("https://www.samsung.com", 301),
+    TestCase::new("https://www.twitter.com", 301),
+    TestCase::new("https://www.facebook.com", 302),
+    TestCase::new("https://www.microsoft.com", 302),
+    TestCase::new("https://www.ibm.com", 303),
+    TestCase::new("https://www.f5.com", 403),
+];
+
+/// perform an HTTP GET request against `uri` using an s2n-tls config with
+/// `security_policy`.
+async fn https_get(
+    uri: &str,
+    security_policy: &Policy,
+) -> Result<Response<Incoming>, hyper_util::client::legacy::Error> {
+    let mut config = Config::builder();
+    config.set_security_policy(security_policy).unwrap();
+
+    let connector = HttpsConnector::new(config.build().unwrap());
+    let client: Client<_, Empty<Bytes>> = Client::builder(TokioExecutor::new()).build(connector);
+
+    let uri = Uri::from_str(uri).unwrap();
+    client.get(uri).await
+}
+
+/// Ensure that s2n-tls is compatible with other http/TLS implementations.
+///
+/// This test uses s2n-tls-hyper to make http requests over a TLS connection to
+/// a number of well known http sites.
+#[test_log::test(tokio::test)]
+async fn http_get_test() -> Result<(), Box<dyn std::error::Error>> {
+    for test_case in TEST_CASES {
+        for policy in [security::DEFAULT, security::DEFAULT_TLS13] {
+            tracing::info!("executing test case {:#?} with {:?}", test_case, policy);
+
+            let response = https_get(test_case.query_target, &policy).await?;
+            let expected_status = StatusCode::from_u16(test_case.expected_status_code).unwrap();
+            assert_eq!(response.status(), expected_status);
+
+            if expected_status == StatusCode::OK {
+                let body = response.into_body().collect().await?.to_bytes();
+                assert!(!body.is_empty());
+            }
+        }
+    }
+
+    Ok(())
+}

bindings/rust/integration/src/network/mod.rs

@@ -0,0 +1,5 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+mod https_client;
+mod tls_client;

bindings/rust/integration/src/network/tls_client.rs

@@ -0,0 +1,95 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use s2n_tls::{config::Config, enums::Version, security::Policy};
+use s2n_tls_tokio::{TlsConnector, TlsStream};
+use tokio::net::TcpStream;
+
+/// Perform a TLS handshake with port 443 of `domain`.
+///
+/// * `domain`: The domain to perform the handshake with
+/// * `security_policy`: The security policy to set on the handshaking client.
+///
+/// Returns an open `TlsStream` if the handshake was successful, otherwise an
+/// `Err``.
+async fn handshake_with_domain(
+    domain: &str,
+    security_policy: &str,
+) -> Result<TlsStream<TcpStream>, Box<dyn std::error::Error>> {
+    tracing::info!("querying {domain} with {security_policy}");
+    const PORT: u16 = 443;
+
+    let mut config = Config::builder();
+    config.set_security_policy(&Policy::from_version(security_policy)?)?;
+
+    let client = TlsConnector::new(config.build()?);
+    // open the TCP stream
+    let stream = TcpStream::connect((domain, PORT)).await?;
+    // complete the TLS handshake
+    Ok(client.connect(domain, stream).await?)
+}
+
+#[cfg(feature = "pq")]
+mod kms_pq {
+    use super::*;
+
+    const DOMAIN: &str = "kms.us-east-1.amazonaws.com";
+
+    // confirm that we successfully negotiate a supported PQ key exchange.
+    //
+    // Note: In the future KMS will deprecate kyber_r3 in favor of ML-KEM.
+    // At that point this test should be updated with a security policy that
+    // supports ML-KEM.
+    #[test_log::test(tokio::test)]
+    async fn pq_handshake() -> Result<(), Box<dyn std::error::Error>> {
+        let tls = handshake_with_domain(DOMAIN, "KMS-PQ-TLS-1-0-2020-07").await?;
+
+        assert_eq!(
+            tls.as_ref().cipher_suite()?,
+            "ECDHE-KYBER-RSA-AES256-GCM-SHA384"
+        );
+        assert_eq!(tls.as_ref().kem_name(), Some("kyber512r3"));
+
+        Ok(())
+    }
+
+    // We want to confirm that non-supported kyber drafts successfully fall
+    // back to a full handshake.
+    #[test_log::test(tokio::test)]
+    async fn early_draft_falls_back_to_classical() -> Result<(), Box<dyn std::error::Error>> {
+        const EARLY_DRAFT_PQ_POLICIES: &[&str] = &[
+            "KMS-PQ-TLS-1-0-2019-06",
+            "PQ-SIKE-TEST-TLS-1-0-2019-11",
+            "KMS-PQ-TLS-1-0-2020-02",
+            "PQ-SIKE-TEST-TLS-1-0-2020-02",
+        ];
+
+        for security_policy in EARLY_DRAFT_PQ_POLICIES {
+            let tls = handshake_with_domain(DOMAIN, security_policy).await?;
+
+            assert_eq!(tls.as_ref().cipher_suite()?, "ECDHE-RSA-AES256-GCM-SHA384");
+            assert_eq!(tls.as_ref().kem_name(), None);
+        }
+        Ok(())
+    }
+}
+
+#[test_log::test(tokio::test)]
+async fn tls_client() -> Result<(), Box<dyn std::error::Error>> {
+    // The akamai request should be in internet_https_client.rs but Akamai
+    // http requests hang indefinitely. This behavior is also observed with
+    // curl and chrome. https://github.com/aws/s2n-tls/issues/4883
+    const DOMAINS: &[&str] = &["www.akamai.com"];
+
+    for domain in DOMAINS {
+        tracing::info!("querying {domain}");
+
+        let tls12 = handshake_with_domain(domain, "default").await?;
+        assert_eq!(tls12.as_ref().actual_protocol_version()?, Version::TLS12);
+
+        let tls13 = handshake_with_domain(domain, "default_tls13").await?;
+        assert_eq!(tls13.as_ref().actual_protocol_version()?, Version::TLS13);
+    }
+
+    Ok(())
+}

bindings/rust/s2n-tls-hyper/Cargo.toml

@@ -23,4 +23,5 @@ http = { version= "1" }
 [dev-dependencies]
 tokio = { version = "1", features = ["macros", "test-util"] }
 http-body-util = "0.1"
+hyper-util = { version = "0.1", features = ["server"] }
 bytes = "1"

bindings/rust/s2n-tls-hyper/tests/common/echo.rs

@@ -0,0 +1,85 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use bytes::Bytes;
+use http::{Request, Response};
+use http_body_util::{combinators::BoxBody, BodyExt};
+use hyper::service::service_fn;
+use hyper_util::rt::{TokioExecutor, TokioIo};
+use s2n_tls::connection::Builder;
+use s2n_tls_tokio::TlsAcceptor;
+use std::{error::Error, future::Future};
+use tokio::net::TcpListener;
+
+async fn echo(
+    req: Request<hyper::body::Incoming>,
+) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
+    Ok(Response::new(req.into_body().boxed()))
+}
+
+async fn serve_echo<B>(
+    tcp_listener: TcpListener,
+    builder: B,
+) -> Result<(), Box<dyn Error + Send + Sync>>
+where
+    B: Builder,
+    <B as Builder>::Output: Unpin + Send + Sync + 'static,
+{
+    let (tcp_stream, _) = tcp_listener.accept().await?;
+    let acceptor = TlsAcceptor::new(builder);
+    let tls_stream = acceptor.accept(tcp_stream).await?;
+    let io = TokioIo::new(tls_stream);
+
+    let server = hyper_util::server::conn::auto::Builder::new(TokioExecutor::new());
+    if let Err(err) = server.serve_connection(io, service_fn(echo)).await {
+        // The hyper client doesn't gracefully terminate by waiting for the server's shutdown.
+        // Instead, the client sends its shutdown and then immediately closes the socket. This can
+        // cause a NotConnected error to be emitted when the server attempts to send its shutdown.
+        //
+        // For now, NotConnected errors are ignored. After the hyper client can be configured to
+        // gracefully shutdown, this exception can be removed:
+        // https://github.com/aws/s2n-tls/issues/4855
+        //
+        // Also, it's possible that a NotConnected error could occur during some operation other
+        // than a shutdown. Ideally, these NotConnected errors wouldn't be ignored. However, it's
+        // not currently possible to distinguish between shutdown vs non-shutdown errors:
+        // https://github.com/aws/s2n-tls/issues/4856
+        if let Some(hyper_err) = err.downcast_ref::<hyper::Error>() {
+            if let Some(source) = hyper_err.source() {
+                if let Some(io_err) = source.downcast_ref::<tokio::io::Error>() {
+                    if io_err.kind() == tokio::io::ErrorKind::NotConnected {
+                        return Ok(());
+                    }
+                }
+            }
+        }
+
+        return Err(err);
+    }
+
+    Ok(())
+}
+
+pub async fn make_echo_request<B, F, Fut>(
+    server_builder: B,
+    send_client_request: F,
+) -> Result<(), Box<dyn Error + Send + Sync>>
+where
+    B: Builder + Send + Sync + 'static,
+    <B as Builder>::Output: Unpin + Send + Sync + 'static,
+    F: FnOnce(u16) -> Fut,
+    Fut: Future<Output = Result<(), Box<dyn Error + Send + Sync>>> + Send + 'static,
+{
+    let listener = TcpListener::bind("127.0.0.1:0").await?;
+    let addr = listener.local_addr()?;
+
+    let mut tasks = tokio::task::JoinSet::new();
+    tasks.spawn(serve_echo(listener, server_builder));
+    tasks.spawn(send_client_request(addr.port()));
+
+    while let Some(res) = tasks.join_next().await {
+        res.unwrap()?;
+    }
+
+    Ok(())
+}

bindings/rust/s2n-tls-hyper/tests/common/mod.rs

@@ -0,0 +1,27 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use s2n_tls::{callbacks::VerifyHostNameCallback, config, error::Error, security::DEFAULT_TLS13};
+
+pub mod echo;
+
+/// NOTE: this certificate and key are used for testing purposes only!
+pub static CERT_PEM: &[u8] =
+    include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../certs/cert.pem"));
+pub static KEY_PEM: &[u8] =
+    include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../certs/key.pem"));
+
+pub fn config() -> Result<config::Builder, Error> {
+    let mut builder = config::Config::builder();
+    builder.set_security_policy(&DEFAULT_TLS13)?;
+    builder.trust_pem(CERT_PEM)?;
+    builder.load_pem(CERT_PEM, KEY_PEM)?;
+    Ok(builder)
+}
+
+pub struct InsecureAcceptAllCertificatesHandler {}
+impl VerifyHostNameCallback for InsecureAcceptAllCertificatesHandler {
+    fn verify_host_name(&self, _host_name: &str) -> bool {
+        true
+    }
+}