From 589f644f42ce608b251dc2c2ecfdda0bf9a542ac Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 2 Oct 2023 10:08:53 -0700 Subject: [PATCH 1/2] client: Add feature flag for BoringSSL TLS ## Motivation Curerently, `kube-client` supports using either Rustls or OpenSSL as the TLS backend. BoringSSL is an alterative TLS implementation, based on a fork of OpenSSL and maintained by Google. Rust bindings for BoringSSL are available in the `boring` crate. Some users of `kube-client` may wish to use BoringSSL as the TLS implementation, rather than OpenSSL or Rustls. This can potentially be implemented in downstream code, on top of `kube-client`'s config module, but it seemed nicer to just provide it in `kube-client`. ## Solution This commit adds support for BoringSSL as a third TLS option. The `boring` crate has a very similar API to the `openssl` crate, so the BoringSSL client code looks quite similar to the OpenSSL client code. I've also attempted to modify the CI configuration to also run tests with BoringSSL. However, I wasn't able to verify that this works without an actual CI run, so it's possible that the E2E test docker environment can't actually build BoringSSL and we'll have to add additional build deps there. I'm happy to follow up on that after a CI run has completed. Signed-off-by: Eliza Weisman --- .github/workflows/ci.yml | 2 +- e2e/Cargo.toml | 1 + kube-client/Cargo.toml | 3 + kube-client/src/client/builder.rs | 13 +++- kube-client/src/client/config_ext.rs | 92 +++++++++++++++++++++++- kube-client/src/client/mod.rs | 14 ++-- kube-client/src/client/tls.rs | 102 +++++++++++++++++++++++++++ kube-client/src/error.rs | 6 ++ kube/Cargo.toml | 1 + 9 files changed, 227 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56296ce75..569fe0d0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -223,7 +223,7 @@ jobs: # Prevent GitHub from cancelling all in-progress jobs when a matrix job fails. fail-fast: false matrix: - tls: [openssl, rustls] + tls: [openssl, rustls, boring] steps: - uses: actions/checkout@v4 - uses: actions/cache@v2 diff --git a/e2e/Cargo.toml b/e2e/Cargo.toml index 492a131f8..e283a27b4 100644 --- a/e2e/Cargo.toml +++ b/e2e/Cargo.toml @@ -22,6 +22,7 @@ latest = ["k8s-openapi/latest"] mk8sv = ["k8s-openapi/v1_23"] rustls = ["kube/rustls-tls"] openssl = ["kube/openssl-tls"] +boring = ["kube/boring-tls"] [dependencies] anyhow = "1.0.44" diff --git a/kube-client/Cargo.toml b/kube-client/Cargo.toml index c19433e47..7cbae9064 100644 --- a/kube-client/Cargo.toml +++ b/kube-client/Cargo.toml @@ -19,6 +19,7 @@ edition = "2021" default = ["client"] rustls-tls = ["rustls", "rustls-pemfile", "hyper-rustls"] openssl-tls = ["openssl", "hyper-openssl"] +boring-tls = ["boring", "hyper-boring"] ws = ["client", "tokio-tungstenite", "rand", "kube-core/ws", "tokio/macros"] oauth = ["client", "tame-oauth"] oidc = ["client", "form_urlencoded"] @@ -38,6 +39,7 @@ rustdoc-args = ["--cfg", "docsrs"] [dependencies] base64 = { version = "0.20.0", optional = true } +boring = { version = "3.1.0", optional = true } chrono = { version = "0.4.23", optional = true, default-features = false } home = { version = "0.5.4", optional = true } serde = { version = "1.0.130", features = ["derive"] } @@ -58,6 +60,7 @@ kube-core = { path = "../kube-core", version = "=0.86.0" } jsonpath_lib = { version = "0.3.0", optional = true } tokio-util = { version = "0.7.0", optional = true, features = ["io", "codec"] } hyper = { version = "0.14.13", optional = true, features = ["client", "http1", "stream", "tcp"] } +hyper-boring = { version = "3.1.0", optional = true } hyper-rustls = { version = "0.24.0", optional = true } tokio-tungstenite = { version = "0.20.0", optional = true } tower = { version = "0.4.13", optional = true, features = ["buffer", "filter", "util"] } diff --git a/kube-client/src/client/builder.rs b/kube-client/src/client/builder.rs index bfaa945c5..c8f7151b9 100644 --- a/kube-client/src/client/builder.rs +++ b/kube-client/src/client/builder.rs @@ -81,13 +81,24 @@ impl TryFrom for ClientBuilder, Response // Current TLS feature precedence when more than one are set: // 1. rustls-tls // 2. openssl-tls + // 3. boring-tls // Create a custom client to use something else. // If TLS features are not enabled, http connector will be used. #[cfg(feature = "rustls-tls")] let connector = config.rustls_https_connector_with_connector(connector)?; #[cfg(all(not(feature = "rustls-tls"), feature = "openssl-tls"))] let connector = config.openssl_https_connector_with_connector(connector)?; - #[cfg(all(not(feature = "rustls-tls"), not(feature = "openssl-tls")))] + #[cfg(all( + not(feature = "rustls-tls"), + not(feature = "openssl-tls"), + feature = "boring-tls" + ))] + let connector = config.boring_https_connector_with_connector(connector)?; + #[cfg(all( + not(feature = "rustls-tls"), + not(feature = "openssl-tls"), + not(feature = "boring-tls") + ))] if auth_layer.is_none() || config.cluster_url.scheme() == Some(&http::uri::Scheme::HTTPS) { // no tls stack situation only works on anonymous auth with http scheme return Err(Error::TlsRequired); diff --git a/kube-client/src/client/config_ext.rs b/kube-client/src/client/config_ext.rs index b0ad0ce5e..ec7c9d9cc 100644 --- a/kube-client/src/client/config_ext.rs +++ b/kube-client/src/client/config_ext.rs @@ -4,7 +4,8 @@ use http::{header::HeaderName, HeaderValue}; use secrecy::ExposeSecret; use tower::{filter::AsyncFilterLayer, util::Either}; -#[cfg(any(feature = "rustls-tls", feature = "openssl-tls"))] use super::tls; +#[cfg(any(feature = "rustls-tls", feature = "openssl-tls"))] +use super::tls; use super::{ auth::Auth, middleware::{AddAuthorizationLayer, AuthLayer, BaseUriLayer, ExtraHeadersLayer}, @@ -143,6 +144,63 @@ pub trait ConfigExt: private::Sealed { #[cfg_attr(docsrs, doc(cfg(feature = "openssl-tls")))] #[cfg(feature = "openssl-tls")] fn openssl_ssl_connector_builder(&self) -> Result; + + /// Create [`hyper_boring::HttpsConnector`] based on config. + /// # Example + /// + /// ```rust + /// # async fn doc() -> Result<(), Box> { + /// # use kube::{client::ConfigExt, Config}; + /// let config = Config::infer().await?; + /// let https = config.boring_https_connector()?; + /// # Ok(()) + /// # } + /// ``` + #[cfg_attr(docsrs, doc(cfg(feature = "boring-tls")))] + #[cfg(feature = "boring-tls")] + fn boring_https_connector(&self) -> Result>; + + /// Create [`hyper_boring::HttpsConnector`] based on config and `connector`. + /// # Example + /// + /// ```rust + /// # async fn doc() -> Result<(), Box> { + /// # use hyper::client::HttpConnector; + /// # use kube::{client::ConfigExt, Config}; + /// let mut http = HttpConnector::new(); + /// http.enforce_http(false); + /// let config = Config::infer().await?; + /// let https = config.boring_https_connector_with_connector(http)?; + /// # Ok(()) + /// # } + /// ``` + #[cfg_attr(docsrs, doc(cfg(feature = "boring-tls")))] + #[cfg(feature = "boring-tls")] + fn boring_https_connector_with_connector( + &self, + connector: hyper::client::HttpConnector, + ) -> Result>; + + /// Create [`boring::ssl::SslConnectorBuilder`] based on config. + /// # Example + /// + /// ```rust + /// # async fn doc() -> Result<(), Box> { + /// # use hyper::client::HttpConnector; + /// # use kube::{client::ConfigExt, Client, Config}; + /// let config = Config::infer().await?; + /// let https = { + /// let mut http = HttpConnector::new(); + /// http.enforce_http(false); + /// let ssl = config.boring_ssl_connector_builder()?; + /// hyper_boring::HttpsConnector::with_connector(http, ssl)? + /// }; + /// # Ok(()) + /// # } + /// ``` + #[cfg_attr(docsrs, doc(cfg(feature = "boring-tls")))] + #[cfg(feature = "boring-tls")] + fn boring_ssl_connector_builder(&self) -> Result; } mod private { @@ -260,6 +318,38 @@ impl ConfigExt for Config { } Ok(https) } + + #[cfg(feature = "boring-tls")] + fn boring_ssl_connector_builder(&self) -> Result { + let identity = self.exec_identity_pem().or_else(|| self.identity_pem()); + // TODO: pass self.tls_server_name for boring + tls::boring_tls::ssl_connector_builder(identity.as_ref(), self.root_cert.as_ref()) + .map_err(|e| Error::BoringTls(tls::boring_tls::Error::CreateSslConnector(e))) + } + + #[cfg(feature = "boring-tls")] + fn boring_https_connector(&self) -> Result> { + let mut connector = hyper::client::HttpConnector::new(); + connector.enforce_http(false); + self.boring_https_connector_with_connector(connector) + } + + #[cfg(feature = "boring-tls")] + fn boring_https_connector_with_connector( + &self, + connector: hyper::client::HttpConnector, + ) -> Result> { + let mut https = + hyper_boring::HttpsConnector::with_connector(connector, self.boring_ssl_connector_builder()?) + .map_err(|e| Error::BoringTls(tls::boring_tls::Error::CreateHttpsConnector(e)))?; + if self.accept_invalid_certs { + https.set_callback(|ssl, _uri| { + ssl.set_verify(boring::ssl::SslVerifyMode::NONE); + Ok(()) + }); + } + Ok(https) + } } impl Config { diff --git a/kube-client/src/client/mod.rs b/kube-client/src/client/mod.rs index 83d87e74a..d752ee517 100644 --- a/kube-client/src/client/mod.rs +++ b/kube-client/src/client/mod.rs @@ -35,12 +35,17 @@ mod config_ext; pub use auth::Error as AuthError; pub use config_ext::ConfigExt; pub mod middleware; -#[cfg(any(feature = "rustls-tls", feature = "openssl-tls"))] mod tls; +#[cfg(any(feature = "rustls-tls", feature = "openssl-tls"))] +mod tls; +#[cfg(feature = "boring-tls")] +pub use tls::boring_tls::Error as BoringTlsError; #[cfg(feature = "openssl-tls")] pub use tls::openssl_tls::Error as OpensslTlsError; -#[cfg(feature = "rustls-tls")] pub use tls::rustls_tls::Error as RustlsTlsError; -#[cfg(feature = "ws")] mod upgrade; +#[cfg(feature = "rustls-tls")] +pub use tls::rustls_tls::Error as RustlsTlsError; +#[cfg(feature = "ws")] +mod upgrade; #[cfg(feature = "oauth")] #[cfg_attr(docsrs, doc(cfg(feature = "oauth")))] @@ -50,7 +55,8 @@ pub use auth::OAuthError; #[cfg_attr(docsrs, doc(cfg(feature = "oidc")))] pub use auth::oidc_errors; -#[cfg(feature = "ws")] pub use upgrade::UpgradeConnectionError; +#[cfg(feature = "ws")] +pub use upgrade::UpgradeConnectionError; pub use builder::{ClientBuilder, DynBody}; diff --git a/kube-client/src/client/tls.rs b/kube-client/src/client/tls.rs index 45785a8c9..ae9b7a248 100644 --- a/kube-client/src/client/tls.rs +++ b/kube-client/src/client/tls.rs @@ -239,3 +239,105 @@ pub mod openssl_tls { Ok(builder) } } + +#[cfg(feature = "boring-tls")] +pub mod boring_tls { + use boring::{ + pkey::PKey, + ssl::{SslConnector, SslConnectorBuilder, SslMethod}, + x509::X509, + }; + use thiserror::Error; + + /// Errors from BoringSSL TLS + #[derive(Debug, Error)] + pub enum Error { + /// Failed to create BoringSSL HTTPS connector + #[error("failed to create BoringSSL HTTPS connector: {0}")] + CreateHttpsConnector(#[source] boring::error::ErrorStack), + + /// Failed to create BoringSSL SSL connector + #[error("failed to create BoringSSL SSL connector: {0}")] + CreateSslConnector(#[source] SslConnectorError), + } + + /// Errors from creating a `SslConnectorBuilder` + #[derive(Debug, Error)] + pub enum SslConnectorError { + /// Failed to build SslConnectorBuilder + #[error("failed to build SslConnectorBuilder: {0}")] + CreateBuilder(#[source] boring::error::ErrorStack), + + /// Failed to deserialize PEM-encoded chain of certificates + #[error("failed to deserialize PEM-encoded chain of certificates: {0}")] + DeserializeCertificateChain(#[source] boring::error::ErrorStack), + + /// Failed to deserialize PEM-encoded private key + #[error("failed to deserialize PEM-encoded private key: {0}")] + DeserializePrivateKey(#[source] boring::error::ErrorStack), + + /// Failed to set private key + #[error("failed to set private key: {0}")] + SetPrivateKey(#[source] boring::error::ErrorStack), + + /// Failed to get a leaf certificate, the certificate chain is empty + #[error("failed to get a leaf certificate, the certificate chain is empty")] + GetLeafCertificate, + + /// Failed to set the leaf certificate + #[error("failed to set the leaf certificate: {0}")] + SetLeafCertificate(#[source] boring::error::ErrorStack), + + /// Failed to append a certificate to the chain + #[error("failed to append a certificate to the chain: {0}")] + AppendCertificate(#[source] boring::error::ErrorStack), + + /// Failed to deserialize DER-encoded root certificate + #[error("failed to deserialize DER-encoded root certificate: {0}")] + DeserializeRootCertificate(#[source] boring::error::ErrorStack), + + /// Failed to add a root certificate + #[error("failed to add a root certificate: {0}")] + AddRootCertificate(#[source] boring::error::ErrorStack), + } + + /// Create `boring::ssl::SslConnectorBuilder` required for `hyper_boring::HttpsConnector`. + pub fn ssl_connector_builder( + identity_pem: Option<&Vec>, + root_certs: Option<&Vec>>, + ) -> Result { + let mut builder = + SslConnector::builder(SslMethod::tls()).map_err(SslConnectorError::CreateBuilder)?; + if let Some(pem) = identity_pem { + let mut chain = X509::stack_from_pem(pem) + .map_err(SslConnectorError::DeserializeCertificateChain)? + .into_iter(); + let leaf_cert = chain.next().ok_or(SslConnectorError::GetLeafCertificate)?; + builder + .set_certificate(&leaf_cert) + .map_err(SslConnectorError::SetLeafCertificate)?; + for cert in chain { + builder + .add_extra_chain_cert(cert) + .map_err(SslConnectorError::AppendCertificate)?; + } + + let pkey = PKey::private_key_from_pem(pem).map_err(SslConnectorError::DeserializePrivateKey)?; + builder + .set_private_key(&pkey) + .map_err(SslConnectorError::SetPrivateKey)?; + } + + if let Some(ders) = root_certs { + for der in ders { + let cert = X509::from_der(der).map_err(SslConnectorError::DeserializeRootCertificate)?; + builder + .cert_store_mut() + .add_cert(cert) + .map_err(SslConnectorError::AddRootCertificate)?; + } + } + + Ok(builder) + } +} diff --git a/kube-client/src/error.rs b/kube-client/src/error.rs index bc15beb2c..d24118fb6 100644 --- a/kube-client/src/error.rs +++ b/kube-client/src/error.rs @@ -71,6 +71,12 @@ pub enum Error { #[error("rustls tls error: {0}")] RustlsTls(#[source] crate::client::RustlsTlsError), + /// Errors from BoringSSL TLS + #[cfg(feature = "boring-tls")] + #[cfg_attr(docsrs, doc(cfg(feature = "boring-tls")))] + #[error("boringssl tls error: {0}")] + BoringTls(#[source] crate::client::BoringTlsError), + /// Missing TLS stacks when TLS is required #[error("TLS required but no TLS stack selected")] TlsRequired, diff --git a/kube/Cargo.toml b/kube/Cargo.toml index 92af9cb6e..013bce1d9 100644 --- a/kube/Cargo.toml +++ b/kube/Cargo.toml @@ -25,6 +25,7 @@ rustls-tls = ["kube-client/rustls-tls"] # alternative features openssl-tls = ["kube-client/openssl-tls"] +boring-tls = ["kube-client/boring-tls"] # auxiliary features ws = ["kube-client/ws", "kube-core/ws"] From c4fcfea66aa4eb8f8cbf502e274e056ba802fd9d Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 2 Oct 2023 10:37:03 -0700 Subject: [PATCH 2/2] client: document TLS features This commit adds some documentation describing how TLS implementations are selected. I figured this would be nice to add. Signed-off-by: Eliza Weisman --- kube-client/src/client/builder.rs | 6 ++++- kube-client/src/client/mod.rs | 42 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/kube-client/src/client/builder.rs b/kube-client/src/client/builder.rs index c8f7151b9..e82be4035 100644 --- a/kube-client/src/client/builder.rs +++ b/kube-client/src/client/builder.rs @@ -64,7 +64,11 @@ impl ClientBuilder { impl TryFrom for ClientBuilder, Response>, BoxError>> { type Error = Error; - /// Builds a default [`ClientBuilder`] stack from a given configuration + /// Builds a default [`ClientBuilder`] stack from a given configuration. + /// + /// The TLS implementation used by the constructed client depends on which + /// crate feature flags are enabled. See [the documentation on configuring + /// TLS](crate::client#configuring-tls) for details. fn try_from(config: Config) -> Result { use std::time::Duration; diff --git a/kube-client/src/client/mod.rs b/kube-client/src/client/mod.rs index d752ee517..dcc91317b 100644 --- a/kube-client/src/client/mod.rs +++ b/kube-client/src/client/mod.rs @@ -7,6 +7,44 @@ //! //! The [`Client`] can also be used with [`Discovery`](crate::Discovery) to dynamically //! retrieve the resources served by the kubernetes API. +//! +//! ## Configuring TLS +//! +//! The Kubernetes client provided by this crate can be configured to use TLS +//! when connecting to the Kubernetes API. A variety of TLS implementations may +//! be used as the backend for `kube-client`'s TLS support, with the choice of +//! TLS backend controlled by crate feature flags. The following TLS backends +//! are available: +//! +//! | TLS backend | Crate feature flag | Description | +//! |:------------|--------------------|-------------| +//! | `` | `` | When no TLS feature flag is enabled, communication with the Kubernetes API is plaintext. | +//! | [Rustls] | `rustls-tls` | [Rustls] is a pure-Rust TLS implementation. | +//! | [OpenSSL] | `openssl-tls` | [OpenSSL] is a popular TLS implementation written in C. This feature uses the [`openssl` crate]'s Rust bindings for OpenSSL. | +//! | [BoringSSL] | `boring-tls` | [BoringSSL] is a fork of OpenSSL maintained by Google. This feature uses the [`boring` crate]'s Rust bindings for BoringSSL. | +//! +//! Since crate feature flags are additive, more than one TLS feature may be +//! enabled at the same time. However, only one TLS backend may actually be +//! selected. Therefore, conflicts are resolved by selecting one TLS backend, +//! with the following order of priority: +//! +//! 1. **rustls-tls**: If the `rustls-tls` feature is enabled, [Rustls] is +//! always used as the TLS implementation, regardless of what other feature +//! flags are enabled. +//! 2. **openssl-tls**: If the `rustls-tls` feature is not enabled, but the +//! `openssl-tls` feature flag is enabled, then [OpenSSL] is used instead of +//! Rustls. +//! 3. **boring-tls**: If neither the `rustls-tls` nor `openssl-tls` features +//! are enabled, [BoringSSL] is used as the TLS backend. +//! 4. **none**: If none of the `rustls-tls`, `openssl-tls`, and `boring-tls` +//! features are enabled, all communication with the Kubernetes API is +//! plaintext. +//! +//! [Rustls]: https://crates.io/crates/rustls +//! [OpenSSL]: https://www.openssl.org/ +//! [`openssl` crate]: https://crates.io/crates/openssl +//! [BoringSSL]: https://github.com/google/boringssl +//! [`boring` crate]: https://crates.io/crates/boring use either::{Either, Left, Right}; use futures::{self, AsyncBufRead, StreamExt, TryStream, TryStreamExt}; use http::{self, Request, Response, StatusCode}; @@ -131,6 +169,10 @@ impl Client { /// /// If you already have a [`Config`] then use [`Client::try_from`](Self::try_from) /// instead. + /// + /// The TLS implementation used by the returned client depends on which + /// crate feature flags are enabled. See [the documentation on configuring + /// TLS](crate::client#configuring-tls) for details. pub async fn try_default() -> Result { Self::try_from(Config::infer().await.map_err(Error::InferConfig)?) }