replace BoringSSL with OpenSSL (#191)

Depends on #190 

Currently, Kubert has the option to use either Rustls or BoringSSL as
the TLS implementation. However, the BoringSSL feature is incomplete, as
it only configures Kubert's server to use BoringSSL, while the client
will still use "whatever `kube-client` is configured to use". This means
that you don't *really* get all-BoringSSL. Meanwhile, using BoringSSL on
the client-side is quite fraught without upstream support in
`kube-client`.

Therefore, this branch rips out the `boring-tls` feature and replaces it
with an `openssl-tls` feature. Now, we can ensure that the client and
server use the same TLS implementation, because `kube-client` already
supports OpenSSL. In addition, I've added new tests for the TLS server,
and changed the CI client tests to run with both TLS clients.

Closes #188

.github/workflows/client.yml

@@ -25,7 +25,11 @@ jobs:
         k8s:
           - v1.21
           - v1.26
-    timeout-minutes: 15
+        tls:
+          - openssl-tls
+          - rustls-tls
+    name: local (k8s ${{ matrix.k8s }}, ${{ matrix.tls }})
+    timeout-minutes: 30 # building with OpenSSL can be quite slow...
     runs-on: ubuntu-latest
     env:
       KUBERT_TEST_CLUSTER_VERSION: ${{ matrix.k8s }}
@@ -34,25 +38,31 @@ jobs:
       - uses: linkerd/dev/actions/setup-rust@v40
       - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
       - run: just fetch
-      - run: just build-examples
-      - run: just test-cluster-create
-      - run: just test-cluster-run-watch-pods --log-level=debug
-      - run: just test-cluster-create-ns
+      - run: just --set features ${{ matrix.tls }} build-examples
+      - run: just --set features ${{ matrix.tls }} test-cluster-create
+      - run: just --set features ${{ matrix.tls }} test-cluster-run-watch-pods --log-level=debug
+      - run: just --set features ${{ matrix.tls }} test-cluster-create-ns
       - name: Run just test-cluster-run-watch-pods with impersonation
         run: |
-          just test-cluster-run-watch-pods --log-level=debug \
+          just --set features ${{ matrix.tls }} \
+            test-cluster-run-watch-pods \
+            --log-level=debug \
             --as=system:serviceaccount:${KUBERT_TEST_NS}:watch-pods \
             --kubeconfig=$HOME/.kube/config
-      - run: just test-lease-build
-      - run: just test-lease
+      - run: just --set features ${{ matrix.tls }} test-lease-build
+      - run: just --set features ${{ matrix.tls }} test-lease
 
   in-cluster:
     strategy:
       matrix:
         k8s:
           - v1.21
           - v1.26
-    timeout-minutes: 10
+        tls:
+          - openssl-tls
+          - rustls-tls
+    name: in-cluster (k8s ${{ matrix.k8s }}, ${{ matrix.tls }})
+    timeout-minutes: 15
     runs-on: ubuntu-latest
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -61,8 +71,8 @@ jobs:
       - uses: linkerd/dev/actions/setup-tools@v40
       - uses: linkerd/dev/actions/setup-rust@v40
       - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
-      - run: just build-examples-image
-      - run: just test-cluster-create
-      - run: just test-cluster-import-examples
-      - run: just test-cluster-create-ns
-      - run: just test-cluster-deploy-watch-pods --log-level=debug
+      - run: just --set features ${{ matrix.tls }} build-examples-image
+      - run: just --set features ${{ matrix.tls }} test-cluster-create
+      - run: just --set features ${{ matrix.tls }} test-cluster-import-examples
+      - run: just --set features ${{ matrix.tls }} test-cluster-create-ns
+      - run: just --set features ${{ matrix.tls }} test-cluster-deploy-watch-pods --log-level=debug

.github/workflows/lint.yml

@@ -25,7 +25,7 @@ jobs:
       - run: just fmt-check
 
   doc:
-    timeout-minutes: 5
+    timeout-minutes: 10
     runs-on: ubuntu-latest
     container: ghcr.io/linkerd/dev:v40-rust
     steps:

deny.toml

@@ -78,6 +78,12 @@ skip-tree = [
     # `parking-lot-core` and `dirs-next` (transitive deps via `kube-client`)
     # depend on incompatible versions of `redox_syscall`.
     { name = "redox_syscall" },
+    # `rcgen` (used to generate test key material in TLS server tests) depends
+    # on a version of the `pem` crate that has diverged substantially from
+    # `kube-client`'s dep on the same crate (v3.0 vs v1.1). however, since
+    # `rcgen` is a test-only dependency, it's not a huge deal if it introduces
+    # duplicate deps, as they won't be present in real life.
+    { name = "rcgen" },
 ]
 
 [sources]

examples/Cargo.toml

@@ -9,6 +9,21 @@ rust-version = "1.60"
 [package.metadata.release]
 release = false
 
+[features]
+default = ["rustls-tls"]
+rustls-tls = ["kubert/rustls-tls"]
+openssl-tls = ["kubert/openssl-tls", "openssl"]
+
+[dependencies.kubert]
+path = "../kubert"
+default-features = false
+features = ["clap", "lease", "runtime"]
+
+[dependencies.openssl]
+version = "0.10.57"
+optional = true
+features = ["vendored"]
+
 [dev-dependencies]
 anyhow = "1"
 chrono = { version = "0.4", default-features = false }
@@ -37,11 +52,6 @@ version = "0.85"
 default-features = false
 features = ["client", "derive", "rustls-tls", "runtime"]
 
-[dev-dependencies.kubert]
-path = "../kubert"
-default-features = false
-features = ["clap", "lease", "runtime", "rustls-tls"]
-
 [dev-dependencies.tokio]
 version = "1"
 features = ["macros", "parking_lot", "rt", "rt-multi-thread", "time"]

examples/Dockerfile

@@ -3,8 +3,11 @@ WORKDIR /kubert
 COPY . .
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     CARGO_NET_RETRY=10 just-cargo fetch
+ARG FEATURES="rustls-tls"
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
-    CARGO_INCREMENTAL=0 just-cargo build --frozen --package=kubert-examples --examples
+    CARGO_INCREMENTAL=0 just-cargo build \
+    --frozen --package=kubert-examples --examples \
+    --no-default-features --features=${FEATURES}
 
 FROM gcr.io/distroless/cc
 COPY --from=build /kubert/target/debug/examples/watch-pods /watch-pods

justfile

@@ -55,10 +55,13 @@ build *args:
 
 build-examples name='':
     @just-cargo build --package=kubert-examples \
-        {{ if name == '' { "--examples" } else { "--example=" + name } }}
+        {{ if name == '' { "--examples" } else { "--example=" + name } }} \
+        {{ _features }}
 
 build-examples-image:
-    docker buildx build . -f examples/Dockerfile --tag=kubert-examples:test --output=type=docker
+    docker buildx build . -f examples/Dockerfile \
+        --tag=kubert-examples:test --output=type=docker \
+        {{ if features != "all" { "--build-arg='FEATURES=" + features + "'" } else { "" } }}
 
 test-cluster-create:
     #!/usr/bin/env bash

kubert/Cargo.toml

@@ -15,11 +15,12 @@ rustls-tls = [
     "rustls-pemfile",
     "kube-client?/rustls-tls",
 ]
-boring-tls = [
-    "boring",
-    "hyper-boring",
-    "tokio-boring",
+openssl-tls = [
+    "openssl",
+    "hyper-openssl",
+    "tokio-openssl",
     "once_cell",
+    "kube-client?/openssl-tls",
 ]
 admin = [
     "ahash",
@@ -134,7 +135,6 @@ ahash = { version = "0.8", optional = true }
 # devcontainer.
 anstyle = { version = "=1.0.0", optional = true }
 backoff = { version = "0.4", features = ["tokio"], optional = true }
-boring = { version = "3.0.4", optional = true }
 bytes = { version = "1", optional = true }
 deflate = { version = "1", optional = true, default-features = false, features = [
     "gzip",
@@ -148,20 +148,21 @@ clap_lex = { version = "=0.5.0", optional = true }
 futures-core = { version = "0.3", optional = true, default-features = false }
 futures-util = { version = "0.3", optional = true, default-features = false }
 hyper = { version = "0.14.17", optional = true, default-features = false }
-hyper-boring = { version = "3.0.4", optional = true }
+hyper-openssl = { version = "0.9.2", optional = true }
 metrics-exporter-prometheus = { version = "0.12.0", optional = true, default-features = false }
 metrics-process = { version = "1.0.11", optional = true }
 once_cell = { version = "1", optional = true }
+openssl = { version = "0.10.57", optional = true, default-features = false }
 parking_lot = { version = "0.12", optional = true }
 pin-project-lite = { version = "0.2", optional = true }
 rustls-pemfile = { version = "1", optional = true }
 thiserror = { version = "1.0.30", optional = true }
 serde = { version = "1", optional = true }
 serde_json = { version = "1", optional = true }
 tokio = { version = "1.17.0", optional = false, default-features = false }
-tokio-boring = { version = "3.0.4", optional = true }
-tokio-util = { version = "0.7", optional = true, default-features = false }
 tokio-rustls = { version = "0.24.1", optional = true, default-features = false }
+tokio-openssl = { version = "0.6.3", optional = true }
+tokio-util = { version = "0.7", optional = true, default-features = false }
 tower-http = { version = "0.4.0", optional = true, default-features = false, features = [
     "map-response-body",
 ] }
@@ -212,6 +213,10 @@ kube = { version = "0.85", default-features = false, features = ["runtime"] }
 tokio-stream = "0.1"
 tokio-test = "0.4"
 tracing-subscriber = { version = "0.3", features = ["ansi"] }
+# used for generating TLS certificates in the server tests.
+rcgen = { version = "0.11.2" }
+# used for creating temporary dirs for TLS certificates in the server tests.
+tempfile = "3.8"
 
 [dev-dependencies.k8s-openapi]
 version = "0.19"

kubert/src/client.rs

@@ -1,10 +1,7 @@
 //! Utilities for configuring a [`kube_client::Client`] from the command line
-
-use hyper::{body::HttpBody, Body, Request, Response};
 pub use kube_client::*;
 use std::path::PathBuf;
 use thiserror::Error;
-use tower::{BoxError, Service, ServiceBuilder};
 
 /// Configures a Kubernetes client
 #[derive(Clone, Debug, Default)]
@@ -52,13 +49,6 @@ pub enum ConfigError {
     /// Indicates that the client could not be initialized
     #[error(transparent)]
     Client(#[from] Error),
-
-    /// Indicates that an error was returned by the BoringSSL TLS
-    /// implementation.
-    #[cfg(feature = "boring-tls")]
-    #[cfg_attr(docsrs, doc(cfg(feature = "boring-tls")))]
-    #[error(transparent)]
-    BoringTls(#[from] boring::error::ErrorStack),
 }
 
 impl ClientArgs {
@@ -69,23 +59,7 @@ impl ClientArgs {
     ///
     /// This is basically equivalent to using `kube_client::Client::try_default`, except that it
     /// supports kubeconfig configuration from the command-line.
-
     pub async fn try_client(self) -> Result<Client, ConfigError> {
-        self.try_client_inner().await
-    }
-
-    // If the `boring-tls` feature flag is enabled, build the client using
-    // BoringSSL, instead of whatever TLS implementation `kube-client` will
-    // use.
-    #[cfg(feature = "boring-tls")]
-    async fn try_client_inner(self) -> Result<Client, ConfigError> {
-        let connector = hyper_boring::HttpsConnector::new()?;
-        let client = hyper::client::Client::builder().build(connector);
-        self.try_from_service(client).await
-    }
-
-    #[cfg(not(feature = "boring-tls"))]
-    async fn try_client_inner(self) -> Result<Client, ConfigError> {
         let client = match self.load_local_config().await {
             Ok(client) => client,
             Err(e) if self.is_customized() => return Err(e),
@@ -95,53 +69,6 @@ impl ClientArgs {
         client.try_into().map_err(Into::into)
     }
 
-    /// Initializes a Kubernetes client from a [`tower::Service`].
-    ///
-    /// This will respect the `$KUBECONFIG` environment variable, but otherwise default to
-    /// `~/.kube/config`. The _current-context_ is used unless `context` is set.
-    ///
-    /// This is basically equivalent to using `kube_client::Client::new`, except that it
-    /// supports kubeconfig configuration from the command-line.
-    pub async fn try_from_service<S, B>(self, svc: S) -> Result<Client, ConfigError>
-    where
-        S: Service<Request<Body>, Response = Response<B>> + Send + Clone + 'static,
-        S::Future: Send + 'static,
-        S::Error: Into<BoxError>,
-        B: HttpBody<Data = bytes::Bytes> + Send + Unpin + 'static,
-        B::Error: Into<BoxError> + Send + Sync,
-    {
-        use kube_client::client::{ClientBuilder, ConfigExt};
-
-        let config = match self.load_local_config().await {
-            Ok(client) => client,
-            Err(e) if self.is_customized() => return Err(e),
-            Err(_) => Config::incluster()?,
-        };
-
-        let stack = ServiceBuilder::new()
-            .layer(config.base_uri_layer())
-            // TODO(eliza): add an equivalent gzip config to the one from kube_client?
-            .option_layer(config.auth_layer()?)
-            .layer(config.extra_headers_layer()?);
-
-        #[cfg(feature = "gzip")]
-        let stack = {
-            use tower_http::{
-                decompression::DecompressionLayer, map_response_body::MapResponseBodyLayer,
-            };
-            stack
-                .layer(DecompressionLayer::new())
-                .layer(MapResponseBodyLayer::new(|body| {
-                    Box::new(HttpBody::map_err(body, Into::into))
-                        as Box<dyn HttpBody<Data = bytes::Bytes, Error = BoxError> + Send + Unpin>
-                }))
-        };
-
-        let svc = stack.service(svc);
-
-        Ok(ClientBuilder::new(svc, config.default_namespace).build())
-    }
-
     /// Indicates whether the command-line arguments attempt to customize the Kubernetes
     /// configuration.
     fn is_customized(&self) -> bool {

kubert/src/lib.rs

@@ -46,19 +46,23 @@
 //! panic when starting the server.
 //!
 //! - **rustls-tls**: Use [`rustls`] as the TLS implementation.
-//! - **boring-tls**: Use [BoringSSL] (via the [`boring`] crate) as the TLS
+//! - **openssl-tls**: Use [OpenSSL] (via the [`openssl`] crate) as the TLS
 //!   implementation. This feature takes priority over the **rustls-tls**
-//!   feature flag. If both are enabled, BoringSSL will be used instead of
+//!   feature flag. If both are enabled, OpenSSL will be used instead of
 //!   Rustls.
 //!
+//! If the `client` feature flag is enabled, these features will also enable the
+//! corresponding feature flags on the [`kube-client`] crate, to configure which
+//! TLS implementation is used by the underlying Kubernetes API client.
+//!
 //! [`kube`]: https://github.com/kube-rs/kube-rs
 //! [Cargo features]: https://doc.rust-lang.org/cargo/reference/features.html
 //! [`clap`]: https://crates.io/crates/clap
 //! [`clap::Parser`]: https://docs.rs/clap/4/clap/trait.Parser.html
 //! [`kube-client`]: https://crates.io/crates/kube-client
 //! [`rustls`]: https://crates.io/crates/rustls
-//! [BoringSSL]: https://github.com/google/boringssl
-//! [`boring`]: https://crates.io/crates/boring
+//! [OpenSSL]: https://www.openssl.org/
+//! [`openssl`]: https://crates.io/crates/openssl
 
 #![deny(warnings, rust_2018_idioms, missing_docs)]
 #![forbid(unsafe_code)]