Put rdkafka driver tests behind a feature flag (#1326)

.cargo/config.toml

@@ -8,6 +8,6 @@ rustflags = [
 linker = "aarch64-linux-gnu-gcc"
 
 [alias]
-windsock = "test --release --bench windsock --features alpha-transforms --"
-windsock-debug = "test --bench windsock --features alpha-transforms --"
+windsock = "test --release --bench windsock --features alpha-transforms,rdkafka-driver-tests --"
+windsock-debug = "test --bench windsock --features alpha-transforms,rdkafka-driver-tests --"
 windsock-cloud-docker = "run --package windsock-cloud-docker --"

.github/workflows/build.yaml

@@ -41,8 +41,10 @@ jobs:
  run: shotover-proxy/build/install_ubuntu_packages.sh
  - name: Install cargo-hack
  run: cargo install cargo-hack --version 0.5.8
+ - name: Ensure that dev tools compiles and has no warnings with no features enabled
+ run: cargo clippy --locked ${{ matrix.cargo_flags }} --all-targets -- -D warnings
  - name: Ensure that shotover-proxy compiles and has no warnings under every possible combination of features
  # some things to explicitly point out:
  # * clippy also reports rustc warnings and errors
- # * clippy --all-targets is not run so we only build the shotover_proxy executable
- run: cargo hack --feature-powerset clippy --locked ${{ matrix.cargo_flags }} -- -D warnings
+ # * clippy --all-targets is not run so we only build the shotover_proxy executable without the tests/benches
+ run: cargo hack --feature-powerset clippy --locked ${{ matrix.cargo_flags }} --package shotover-proxy -- -D warnings

shotover-proxy/Cargo.toml

@@ -19,8 +19,6 @@ clap.workspace = true
 rstest = "0.18.0"
 cassandra-cpp = { version = "2.0.0", default-features = false }
 test-helpers = { path = "../test-helpers" }
-#rdkafka = { version = "0.32", features = ["cmake-build"] }
-rdkafka = { branch = "updating_librdkafka_to_v2.1.1", git = "https://github.com/shotover/rust-rdkafka", features = ["cmake-build"] }
 redis.workspace = true
 chacha20poly1305.workspace = true
 serde.workspace = true
@@ -59,6 +57,7 @@ inferno = { version = "0.11.15", default-features = false, features = ["multithr
 # Include WIP alpha transforms in the public API
 alpha-transforms = ["shotover/alpha-transforms"]
 cassandra-cpp-driver-tests = ["test-helpers/cassandra-cpp-driver-tests"]
+rdkafka-driver-tests = ["test-helpers/rdkafka-driver-tests"]
 
 [[bench]]
 name = "windsock"

shotover-proxy/benches/windsock/kafka.rs

@@ -6,14 +6,14 @@ use anyhow::Result;
 use async_trait::async_trait;
 use aws_throwaway::Ec2Instance;
 use futures::StreamExt;
-use rdkafka::config::ClientConfig;
-use rdkafka::consumer::{Consumer, StreamConsumer};
-use rdkafka::producer::{FutureProducer, FutureRecord};
-use rdkafka::util::Timeout;
 use std::path::Path;
 use std::sync::Arc;
 use std::{collections::HashMap, time::Duration};
 use test_helpers::docker_compose::docker_compose;
+use test_helpers::rdkafka::config::ClientConfig;
+use test_helpers::rdkafka::consumer::{Consumer, StreamConsumer};
+use test_helpers::rdkafka::producer::{FutureProducer, FutureRecord};
+use test_helpers::rdkafka::util::Timeout;
 use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle, time::Instant};
 use windsock::{Bench, BenchParameters, Profiling, Report};
 

shotover-proxy/benches/windsock/main.rs

@@ -1,13 +1,15 @@
 mod aws;
 mod cassandra;
 mod common;
+#[cfg(feature = "rdkafka-driver-tests")]
 mod kafka;
 mod profilers;
 mod redis;
 mod shotover;
 
 use crate::cassandra::*;
 use crate::common::*;
+#[cfg(feature = "rdkafka-driver-tests")]
 use crate::kafka::*;
 use crate::redis::*;
 use std::path::Path;
@@ -69,7 +71,7 @@ fn main() {
  )) as Box<dyn Bench>)
  },
  );
-
+ #[cfg(feature = "rdkafka-driver-tests")]
  let kafka_benches = itertools::iproduct!(
  [
  Shotover::None,
@@ -79,6 +81,8 @@ fn main() {
  [Size::B1, Size::KB1, Size::KB100,]
  )
  .map(|(shotover, size)| Box::new(KafkaBench::new(shotover, size)) as Box<dyn Bench>);
+ #[cfg(not(feature = "rdkafka-driver-tests"))]
+ let kafka_benches = std::iter::empty();
 
  let redis_benches = itertools::iproduct!(
  [RedisTopology::Cluster3, RedisTopology::Single],

shotover-proxy/tests/kafka_int_tests/mod.rs

@@ -1,14 +1,19 @@
 #[cfg(feature = "alpha-transforms")]
+#[cfg(feature = "rdkafka-driver-tests")]
 use crate::shotover_process;
 #[cfg(feature = "alpha-transforms")]
+#[cfg(feature = "rdkafka-driver-tests")]
 use std::time::Duration;
 #[cfg(feature = "alpha-transforms")]
+#[cfg(feature = "rdkafka-driver-tests")]
 use test_helpers::docker_compose::docker_compose;
 
 #[cfg(feature = "alpha-transforms")]
+#[cfg(feature = "rdkafka-driver-tests")]
 mod test_cases;
 
 #[cfg(feature = "alpha-transforms")]
+#[cfg(feature = "rdkafka-driver-tests")]
 #[tokio::test]
 async fn passthrough_standard() {
  let _docker_compose =
@@ -27,6 +32,7 @@ async fn passthrough_standard() {
  .expect("Shotover did not shutdown within 10s");
 }
 
+#[cfg(feature = "rdkafka-driver-tests")]
 #[cfg(feature = "alpha-transforms")]
 #[tokio::test]
 async fn passthrough_encode() {
@@ -41,6 +47,7 @@ async fn passthrough_encode() {
  shotover.shutdown_and_then_consume_events(&[]).await;
 }
 
+#[cfg(feature = "rdkafka-driver-tests")]
 #[cfg(feature = "alpha-transforms")]
 #[tokio::test]
 async fn cluster_single_shotover() {
@@ -59,6 +66,7 @@ async fn cluster_single_shotover() {
  .expect("Shotover did not shutdown within 10s");
 }
 
+#[cfg(feature = "rdkafka-driver-tests")]
 #[cfg(feature = "alpha-transforms")]
 #[tokio::test]
 async fn cluster_multi_shotover() {

shotover-proxy/tests/kafka_int_tests/test_cases.rs

@@ -1,8 +1,8 @@
-use rdkafka::config::ClientConfig;
-use rdkafka::consumer::{Consumer, StreamConsumer};
-use rdkafka::producer::{FutureProducer, FutureRecord};
-use rdkafka::Message;
 use std::time::Duration;
+use test_helpers::rdkafka::config::ClientConfig;
+use test_helpers::rdkafka::consumer::{Consumer, StreamConsumer};
+use test_helpers::rdkafka::producer::{FutureProducer, FutureRecord};
+use test_helpers::rdkafka::Message;
 
 async fn produce_consume(brokers: &str, topic_name: &str) {
  let producer: FutureProducer = ClientConfig::new()

test-helpers/Cargo.toml

@@ -8,6 +8,7 @@ license = "Apache-2.0"
 
 [features]
 cassandra-cpp-driver-tests = ["cassandra-cpp"]
+rdkafka-driver-tests = ["rdkafka"]
 
 [dependencies]
 tracing.workspace = true
@@ -32,3 +33,5 @@ serde_yaml.workspace = true
 anyhow.workspace = true
 rcgen.workspace = true
 docker-compose-runner = "0.2.0"
+#rdkafka = { version = "0.32", features = ["cmake-build"] }
+rdkafka = { branch = "updating_librdkafka_to_v2.1.1", git = "https://github.com/shotover/rust-rdkafka", features = ["cmake-build"], optional = true }

test-helpers/src/lib.rs

@@ -8,6 +8,9 @@ pub mod shotover_process;
 use anyhow::{anyhow, Result};
 use subprocess::{Exec, Redirection};
 
+#[cfg(feature = "rdkafka-driver-tests")]
+pub use rdkafka;
+
 /// Runs a command and returns the output as a string.
 ///
 /// Both stderr and stdout are returned in the result.