From 6df4d1716f18062a9786681301e230f1a7a9e529 Mon Sep 17 00:00:00 2001 From: Lucas Kent Date: Wed, 1 Feb 2023 15:55:17 +1100 Subject: [PATCH] prevent recompiling by properly specifying build.rs (#1005) --- Cargo.lock | 54 +++++++++++++++++++ shotover-proxy/Cargo.toml | 1 + shotover-proxy/build.rs | 6 --- shotover-proxy/tests/kafka_int_tests/mod.rs | 11 ++++ .../tests/kafka_int_tests/test_cases.rs | 32 +++++++++++ shotover-proxy/tests/lib.rs | 5 +- .../kafka-simple/docker-compose.yaml | 19 +++++++ test-helpers/Cargo.toml | 1 + test-helpers/build.rs | 1 + test-helpers/src/docker_compose.rs | 16 +++++- tokio-bin-process/build.rs | 1 + 11 files changed, 138 insertions(+), 9 deletions(-) delete mode 100644 shotover-proxy/build.rs create mode 100644 shotover-proxy/tests/kafka_int_tests/mod.rs create mode 100644 shotover-proxy/tests/kafka_int_tests/test_cases.rs create mode 100644 shotover-proxy/tests/test-configs/kafka-simple/docker-compose.yaml diff --git a/Cargo.lock b/Cargo.lock index 90ba625d1..5a03e2951 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -550,6 +550,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "cmake" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db34956e100b30725f2eb215f90d4871051239535632f84fea3bc92722c66b7c" +dependencies = [ + "cc", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -1537,6 +1546,18 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +[[package]] +name = "libz-sys" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "link-cplusplus" version = "1.0.8" @@ -2405,6 +2426,37 @@ dependencies = [ "yasna", ] +[[package]] +name = "rdkafka" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd7c5d6d17442bcb9f943aae96d67d98c6d36af60442dd5da62aaa7fcbb25c48" +dependencies = [ + "futures-channel", + "futures-util", + "libc", + "log", + "rdkafka-sys", + "serde", + "serde_derive", + "serde_json", + "slab", + "tokio", +] + +[[package]] +name = "rdkafka-sys" +version = "4.3.0+1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d222a401698c7f2010e3967353eae566d9934dcda49c29910da922414ab4e3f4" +dependencies = [ + "cmake", + "libc", + "libz-sys", + "num_enum", + "pkg-config", +] + [[package]] name = "redis" version = "0.22.2" @@ -3020,6 +3072,7 @@ dependencies = [ "rand", "rand_distr", "rayon", + "rdkafka", "redis", "redis-protocol", "reqwest", @@ -3241,6 +3294,7 @@ dependencies = [ "tokio-io-timeout", "tokio-openssl", "tracing", + "tracing-subscriber", "uuid", ] diff --git a/shotover-proxy/Cargo.toml b/shotover-proxy/Cargo.toml index b8493e032..ce6c1dda9 100644 --- a/shotover-proxy/Cargo.toml +++ b/shotover-proxy/Cargo.toml @@ -93,6 +93,7 @@ nix = "0.26.0" reqwest = "0.11.6" cdrs-tokio = { git = "https://github.com/krojew/cdrs-tokio", branch = "8.0-dev" } rstest = "0.16.0" +rdkafka = { version = "0.29", features = ["cmake-build"] } [[bench]] name = "benches" diff --git a/shotover-proxy/build.rs b/shotover-proxy/build.rs deleted file mode 100644 index cfb768819..000000000 --- a/shotover-proxy/build.rs +++ /dev/null @@ -1,6 +0,0 @@ -use std::env; - -fn main() { - let profile = env::var("PROFILE").unwrap(); - println!("cargo:rustc-env=PROFILE={profile}"); -} diff --git a/shotover-proxy/tests/kafka_int_tests/mod.rs b/shotover-proxy/tests/kafka_int_tests/mod.rs new file mode 100644 index 000000000..6087ce9c1 --- /dev/null +++ b/shotover-proxy/tests/kafka_int_tests/mod.rs @@ -0,0 +1,11 @@ +use serial_test::serial; +use test_helpers::docker_compose::DockerCompose; + +mod test_cases; + +#[tokio::test] +#[serial] +async fn basic() { + let _docker_compose = DockerCompose::new("tests/test-configs/kafka-simple/docker-compose.yaml"); + test_cases::basic().await; +} diff --git a/shotover-proxy/tests/kafka_int_tests/test_cases.rs b/shotover-proxy/tests/kafka_int_tests/test_cases.rs new file mode 100644 index 000000000..8a443907b --- /dev/null +++ b/shotover-proxy/tests/kafka_int_tests/test_cases.rs @@ -0,0 +1,32 @@ +use rdkafka::config::ClientConfig; +use rdkafka::message::{Header, OwnedHeaders}; +use rdkafka::producer::{FutureProducer, FutureRecord}; +use std::time::Duration; + +async fn produce(brokers: &str, topic_name: &str) { + let producer: &FutureProducer = &ClientConfig::new() + .set("bootstrap.servers", brokers) + .set("message.timeout.ms", "5000") + .create() + .expect("Producer creation error"); + + let delivery_status = producer + .send( + FutureRecord::to(topic_name) + .payload("Message") + .key("Key") + .headers(OwnedHeaders::new().insert(Header { + key: "header_key", + value: Some("header_value"), + })), + Duration::from_secs(0), + ) + .await + .unwrap(); + + assert_eq!(delivery_status, (0, 0)); +} + +pub async fn basic() { + produce("localhost:9092", "foo").await; +} diff --git a/shotover-proxy/tests/lib.rs b/shotover-proxy/tests/lib.rs index 264b17cf4..f4fd1ef15 100644 --- a/shotover-proxy/tests/lib.rs +++ b/shotover-proxy/tests/lib.rs @@ -1,6 +1,7 @@ mod cassandra_int_tests; pub mod codec; mod examples; +mod kafka_int_tests; mod redis_int_tests; -pub mod runner; -pub mod transforms; +mod runner; +mod transforms; diff --git a/shotover-proxy/tests/test-configs/kafka-simple/docker-compose.yaml b/shotover-proxy/tests/test-configs/kafka-simple/docker-compose.yaml new file mode 100644 index 000000000..bf7cdd857 --- /dev/null +++ b/shotover-proxy/tests/test-configs/kafka-simple/docker-compose.yaml @@ -0,0 +1,19 @@ +version: "3" +services: + kafka: + image: 'bitnami/kafka:3.3.2' + ports: + - '9092:9092' + environment: + - KAFKA_ENABLE_KRAFT=yes + - KAFKA_CFG_PROCESS_ROLES=broker,controller + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + - KAFKA_BROKER_ID=1 + - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:9093 + - ALLOW_PLAINTEXT_LISTENER=yes + volumes: + - type: tmpfs + target: /bitnami/kafka diff --git a/test-helpers/Cargo.toml b/test-helpers/Cargo.toml index f92b19dab..18065b10d 100644 --- a/test-helpers/Cargo.toml +++ b/test-helpers/Cargo.toml @@ -33,3 +33,4 @@ tokio-openssl = "0.6.2" inferno = "0.11.13" itertools = "0.10.1" reqwest = "0.11.6" +tracing-subscriber = "0.3.16" diff --git a/test-helpers/build.rs b/test-helpers/build.rs index cfb768819..8484a5049 100644 --- a/test-helpers/build.rs +++ b/test-helpers/build.rs @@ -3,4 +3,5 @@ use std::env; fn main() { let profile = env::var("PROFILE").unwrap(); println!("cargo:rustc-env=PROFILE={profile}"); + println!("cargo:rerun-if-env-changed=PROFILE"); } diff --git a/test-helpers/src/docker_compose.rs b/test-helpers/src/docker_compose.rs index 225e62048..a35782055 100644 --- a/test-helpers/src/docker_compose.rs +++ b/test-helpers/src/docker_compose.rs @@ -9,6 +9,15 @@ use std::time::{self, Duration}; use std::{env, path::Path}; use subprocess::{Exec, Redirection}; use tracing::trace; +use tracing_subscriber::fmt::TestWriter; + +fn setup_tracing_subscriber_for_test_logic() { + tracing_subscriber::fmt() + .with_writer(TestWriter::new()) + .with_env_filter("warn") + .try_init() + .ok(); +} /// Runs a command and returns the output as a string. /// @@ -17,7 +26,6 @@ use tracing::trace; /// # Arguments /// * `command` - The system command to run /// * `args` - An array of command line arguments for the command -/// pub fn run_command(command: &str, args: &[&str]) -> Result { trace!("executing {}", command); let data = Exec::cmd(command) @@ -59,6 +67,8 @@ impl DockerCompose { /// # Panics /// * Will panic if docker-compose is not installed pub fn new(file_path: &str) -> Self { + setup_tracing_subscriber_for_test_logic(); + if let Err(ErrorKind::NotFound) = Command::new("docker-compose") .output() .map_err(|e| e.kind()) @@ -164,6 +174,10 @@ impl DockerCompose { name: "shotover-int-tests/cassandra:3.11.13", log_regex_to_wait_for: r"Startup complete", }, + Image { + name: "bitnami/kafka:3.3.2", + log_regex_to_wait_for: r"Kafka Server started", + }, ]; let services: Vec = self diff --git a/tokio-bin-process/build.rs b/tokio-bin-process/build.rs index cfb768819..8484a5049 100644 --- a/tokio-bin-process/build.rs +++ b/tokio-bin-process/build.rs @@ -3,4 +3,5 @@ use std::env; fn main() { let profile = env::var("PROFILE").unwrap(); println!("cargo:rustc-env=PROFILE={profile}"); + println!("cargo:rerun-if-env-changed=PROFILE"); }