Add skeleton for automated performance testing in CI

.github/workflows/performance.yml

@@ -1,58 +1,44 @@
 name: performance
 on:
   workflow_dispatch:
-  # TODO(hubcio): uncomment when the workflow is ready
-  # pull_request:
-  #   branches:
-  #     - master
-  # push:
-  #   branches:
-  #     - master
+  push:
+    branches:
+      - master
 
 jobs:
   run_benchmarks:
     runs-on: performance
     env:
       IGGY_CI_BUILD: true
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+
+      - name: Setup rust toolchain
+        uses: actions-rs/toolchain@v1
         with:
-          ref: master
-          fetch-depth: 0
+          toolchain: stable
+          override: true
 
       - name: Cache cargo & target directories
         uses: Swatinem/rust-cache@v2
         with:
           key: "v2"
 
-      - name: Build iggy-server
-        run: cargo build --release
-
-      - uses: JarvusInnovations/background-action@v1
-        name: Run iggy-server in background
+      - name: Checkout code
+        uses: actions/checkout@v4
         with:
-          run: |
-            target/release/iggy-server &
-          wait-on: tcp:localhost:8090
-          wait-for: 1m
-          log-output: true
-          log-output-if: timeout
-          tail: true
-
-      # Write 10 GB of data to the disk (10 producers * 1000 batches * 1000 messages in batch * 1000 bytes per message)
-      - name: Run send bench
-        timeout-minutes: 1
-        run: target/release/iggy-bench --verbose --warmup-time 0 send tcp
+          fetch-depth: 0
 
-      # Read 10 GB of data from the disk
-      - name: Run poll bench
-        timeout-minutes: 1
-        run: target/release/iggy-bench --verbose poll tcp
+      - name: Verify working directory
+        run: |
+          pwd
+          ls -la
 
-      - name: Stop iggy-server
-        timeout-minutes: 1
-        run: pkill -15 iggy-server && while pgrep -l iggy-server; do sleep 2; done;
+      - name: Run predefined benchmarks
+        timeout-minutes: 60
+        run: ./scripts/performance/run-standard-performance-suite.sh
 
-      - name: Print iggy-server logs
-        run: cat local_data/logs/iggy*
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance_results
+          path: performance_results*

.gitignore

@@ -7,3 +7,4 @@ bench_*
 /sdk/errors_table/
 /rust-clippy-results.sarif
 /Cross.toml
+/performance_results*

bench/Cargo.toml

@@ -7,6 +7,8 @@ edition = "2021"
 async-trait = "0.1.82"
 clap = { version = "4.5.17", features = ["derive"] }
 colored = "2.0.4"
+csv = "1.3.1"
+derive-new = "0.7.0"
 derive_more = "1.0.0"
 figlet-rs = "0.1.5"
 futures = "0.3.30"

bench/src/args/common.rs

@@ -39,6 +39,13 @@ pub struct IggyBenchArgs {
     /// Skip server start
     #[arg(long, short = 'k', default_value_t = DEFAULT_SKIP_SERVER_START)]
     pub skip_server_start: bool,
+
+    /// Output directory, in which the benchmark results will be stored as `csv` and `toml` files.
+    /// Sample from the benchmark will be stored in a `csv` file on per-actor manner - each
+    /// producer/consumer will have its own file.
+    /// Actor summary, benchmark summary and parameters will be stored in a TOML file.
+    #[arg(long, short = 'o', default_value = None)]
+    pub output_directory: Option<String>,
 }
 
 fn validate_server_executable_path(v: &str) -> Result<String, String> {
@@ -133,4 +140,8 @@ impl IggyBenchArgs {
     pub fn warmup_time(&self) -> IggyDuration {
         self.warmup_time
     }
+
+    pub fn output_directory(&self) -> Option<String> {
+        self.output_directory.clone()
+    }
 }

bench/src/args/kind.rs

@@ -4,7 +4,7 @@ use super::props::BenchmarkKindProps;
 use super::transport::BenchmarkTransportCommand;
 use super::{common::IggyBenchArgs, simple::BenchmarkKind};
 use clap::{error::ErrorKind, CommandFactory, Parser, Subcommand};
-use core::panic;
+use serde::Serialize;
 use std::num::NonZeroU32;
 
 #[derive(Subcommand, Debug)]
@@ -97,7 +97,7 @@ impl BenchmarkKindProps for BenchmarkKindCommand {
 }
 
 /// Sending (writing) benchmark
-#[derive(Parser, Debug)]
+#[derive(Parser, Debug, Serialize, Clone)]
 pub struct SendArgs {
     #[command(subcommand)]
     pub transport: BenchmarkTransportCommand,
@@ -153,7 +153,7 @@ impl BenchmarkKindProps for SendArgs {
     }
 
     fn consumers(&self) -> u32 {
-        panic!("")
+        0
     }
 
     fn producers(&self) -> u32 {
@@ -173,7 +173,7 @@ impl BenchmarkKindProps for SendArgs {
     }
 
     fn number_of_consumer_groups(&self) -> u32 {
-        panic!("No consumer groups in send benchmark");
+        0
     }
 
     fn validate(&self) {
@@ -192,7 +192,7 @@ impl BenchmarkKindProps for SendArgs {
 }
 
 /// Sending and Polling benchmark with consumer group
-#[derive(Parser, Debug)]
+#[derive(Parser, Debug, Clone)]
 pub struct ConsumerGroupArgs {
     #[command(subcommand)]
     pub transport: BenchmarkTransportCommand,
@@ -240,23 +240,23 @@ impl BenchmarkKindProps for ConsumerGroupArgs {
     }
 
     fn number_of_partitions(&self) -> u32 {
-        panic!("No partitions in consumer group benchmark");
+        0
     }
 
     fn consumers(&self) -> u32 {
         self.consumers.get()
     }
 
     fn producers(&self) -> u32 {
-        panic!("No producers in consumer group");
+        0
     }
 
     fn disable_parallel_producer_streams(&self) -> bool {
-        panic!("No parallel producer for consumer group");
+        false
     }
 
     fn disable_parallel_consumer_streams(&self) -> bool {
-        panic!("No parallel consumer for consumer group");
+        false
     }
 
     fn transport_command(&self) -> &BenchmarkTransportCommand {
@@ -290,7 +290,7 @@ impl BenchmarkKindProps for ConsumerGroupArgs {
 }
 
 /// Polling (reading) benchmark
-#[derive(Parser, Debug)]
+#[derive(Parser, Debug, Clone, Serialize)]
 pub struct PollArgs {
     #[command(subcommand)]
     pub transport: BenchmarkTransportCommand,
@@ -350,7 +350,7 @@ impl BenchmarkKindProps for PollArgs {
     }
 
     fn producers(&self) -> u32 {
-        panic!("")
+        0
     }
 
     fn disable_parallel_producer_streams(&self) -> bool {
@@ -366,7 +366,7 @@ impl BenchmarkKindProps for PollArgs {
     }
 
     fn number_of_consumer_groups(&self) -> u32 {
-        panic!("No consumer groups in poll benchmark");
+        0
     }
 
     fn validate(&self) {
@@ -385,7 +385,7 @@ impl BenchmarkKindProps for PollArgs {
 }
 
 /// Parallel sending and polling benchmark
-#[derive(Parser, Debug)]
+#[derive(Parser, Debug, Clone)]
 pub struct SendAndPollArgs {
     #[command(subcommand)]
     pub transport: BenchmarkTransportCommand,
@@ -469,7 +469,7 @@ impl BenchmarkKindProps for SendAndPollArgs {
     }
 
     fn number_of_consumer_groups(&self) -> u32 {
-        panic!("No consumer groups in send and poll benchmark");
+        0
     }
 
     fn validate(&self) {

bench/src/args/mod.rs

@@ -1,8 +1,8 @@
 pub mod common;
+pub mod kind;
 pub mod simple;
 
 mod defaults;
 mod examples;
-mod kind;
 mod props;
 mod transport;

bench/src/args/transport.rs

@@ -2,15 +2,30 @@ use super::defaults::*;
 use super::props::BenchmarkTransportProps;
 use clap::{Parser, Subcommand};
 use integration::test_server::Transport;
+use serde::{Serialize, Serializer};
 use std::num::NonZeroU32;
 
-#[derive(Subcommand, Debug)]
+#[derive(Subcommand, Debug, Clone)]
 pub enum BenchmarkTransportCommand {
     Http(HttpArgs),
     Tcp(TcpArgs),
     Quic(QuicArgs),
 }
 
+impl Serialize for BenchmarkTransportCommand {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let variant_str = match self {
+            BenchmarkTransportCommand::Http(_) => "http",
+            BenchmarkTransportCommand::Tcp(_) => "tcp",
+            BenchmarkTransportCommand::Quic(_) => "quic",
+        };
+        serializer.serialize_str(variant_str)
+    }
+}
+
 impl BenchmarkTransportProps for BenchmarkTransportCommand {
     fn transport(&self) -> &Transport {
         self.inner().transport()
@@ -41,7 +56,7 @@ impl BenchmarkTransportProps for BenchmarkTransportCommand {
     }
 }
 
-#[derive(Parser, Debug)]
+#[derive(Parser, Debug, Clone)]
 pub struct HttpArgs {
     /// Address of the HTTP iggy-server
     #[arg(long, default_value_t = DEFAULT_HTTP_SERVER_ADDRESS.to_owned())]
@@ -74,7 +89,7 @@ impl BenchmarkTransportProps for HttpArgs {
     }
 }
 
-#[derive(Parser, Debug)]
+#[derive(Parser, Debug, Clone)]
 pub struct TcpArgs {
     /// Address of the TCP iggy-server
     #[arg(long, default_value_t = DEFAULT_TCP_SERVER_ADDRESS.to_owned())]
@@ -107,7 +122,7 @@ impl BenchmarkTransportProps for TcpArgs {
     }
 }
 
-#[derive(Parser, Debug)]
+#[derive(Parser, Debug, Clone)]
 pub struct QuicArgs {
     /// Address to which the QUIC client will bind
     #[arg(long, default_value_t = DEFAULT_QUIC_CLIENT_ADDRESS.to_owned())]