Improve shotover docs for custom transforms (#1124)

shotover-proxy/Cargo.toml

@@ -8,13 +8,13 @@ license = "Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-anyhow.workspace = true
-tokio.workspace = true
-clap.workspace = true
 shotover = { path = "../shotover" }
-tracing.workspace = true
 
 [dev-dependencies]
+anyhow.workspace = true
+tokio.workspace = true
+tracing.workspace = true
+clap.workspace = true
 criterion = { git = "https://github.com/shotover/criterion.rs", branch = "0.4.0-bench_with_input_fn", features = ["async_tokio"] }
 serial_test = "2.0.0"
 rstest = "0.17.0"

shotover/src/codec/mod.rs

@@ -1,3 +1,5 @@
+//! Codec types to use for connecting to a DB in a sink transform
+
 use crate::message::Messages;
 use cassandra_protocol::compression::Compression;
 use core::fmt;

shotover/src/frame/mod.rs

@@ -1,3 +1,5 @@
+//! parsed AST-like representations of messages
+
 use crate::{codec::CodecState, message::ProtocolType};
 use anyhow::{anyhow, Result};
 use bytes::Bytes;

shotover/src/lib.rs

@@ -1,25 +1,35 @@
-//! The data layer proxy.
+//! This library allows the creation of custom shotover transforms.
 //!
-//! Below are the main areas that you should be looking at to extend and work with Shotover.
+//! There are two consumers of this library:
+//! ## Custom Transforms
 //!
-//! Creating a transform is largely just implementing the [`transforms::Transform`] trait and registering it with
-//! the [`transforms::Transforms`] enum.
+//! To create a custom transform you need to implement these traits:
+//! * [`transforms::TransformConfig`] - Defines what configuration fields the transform has in the `topology.yaml`.
+//! * [`transforms::TransformBuilder`] - Defines how to build the Transform for a new incoming connection. Only one instance is created over shotovers runtime.
+//! * [`transforms::Transform`] - Defines the transformation logic of the transform. A new instance is created per incoming connection.
 //!
-//! To allow your [`transforms::Transform`] to be configurable in Shotover config files you will need to create
-//! a serializable config struct and implement [`transforms::TransformConfig`] trait.
+//! Simple transforms can implement all of these onto a single struct but generally you need seperate structs for each.
 //!
-//! ## Messages
-//! * [`message::Message`], the main struct that carries database queries/frames around in Shotover.
+//! ## The shotover binary
+//! All custom transforms the user wants to use are statically compiled into a single binary.
+//! The crate for this binary is very simple, it just consists of a `main.rs` like:
+//!
+//! ```no_run
+//! # mod transform_crate {
+//! # pub type TransformConfig = shotover::transforms::null::NullSinkConfig;
+//! # }
+//! shotover::import_transform!(transform_crate::TransformConfig);
+//!
+//! fn main() {
+//! shotover::runner::Shotover::new().run_block();
+//! }
+//! ```
 //!
-//! ## Transform
-//! * [`transforms::Wrapper`], used to wrap messages as they traverse the [`transforms::Transform`] chain.
-//! * [`transforms::Transform`], the main extension trait for adding your own behaviour to Shotover.
-//! * [`transforms::Transforms`], the enum to register with (add a variant) for enabling your own transform.
-//! * [`transforms::TransformConfig`], the trait to implement for configuring your own transform.
 
 // Accidentally printing would break json log output
 #![deny(clippy::print_stdout)]
 #![deny(clippy::print_stderr)]
+#![allow(clippy::needless_doctest_main)]
 
 pub mod codec;
 mod config;
@@ -36,6 +46,8 @@ pub mod tls;
 mod tracing_panic_handler;
 pub mod transforms;
 
+/// Imports a custom transform into the shotover binary.
+///
 /// When a custom transform is defined in its own crate, typetag wont kick in unless there is some kind of `use crate_name::CustomTransformConfig`.
 /// This macro does that for you while making it clear that the `use` is a little bit magic.
 /// It also performs some type checks to ensure that you are actually importing an implementer of [`transforms::TransformConfig`].

shotover/src/message/mod.rs

@@ -1,3 +1,5 @@
+//! Message and supporting types - used to hold a message/query/result going between the client and database
+
 use crate::codec::kafka::RequestHeader;
 use crate::codec::CodecState;
 use crate::frame::cassandra::Tracing;
@@ -48,15 +50,16 @@ impl From<&ProtocolType> for CodecState {
 
 pub type Messages = Vec<Message>;
 
-/// The Message type is designed to effeciently abstract over the message being in various states of processing.
+/// Message holds a single message/query/result going between the client and database.
+/// It is designed to efficiently abstract over the message being in various states of processing.
 ///
 /// Usually a message is received and starts off containing just raw bytes (or possibly raw bytes + frame)
 /// This can be immediately sent off to the destination without any processing cost.
 ///
-/// However if a transform wants to query the contents of the message it must call `Message::frame()` which will cause the raw bytes to be processed into a raw bytes + Frame.
+/// However if a transform wants to query the contents of the message it must call [`Message::frame`] which will cause the raw bytes to be processed into a raw bytes + Frame.
 /// The first call to frame has an expensive one time cost.
 ///
-/// The transform may also go one step further and modify the message's Frame + call `Message::invalidate_cache()`.
+/// The transform may also go one step further and modify the message's Frame + call [`Message::invalidate_cache`].
 /// This results in an expensive cost to reassemble the message bytes when the message is sent to the destination.
 #[derive(PartialEq, Debug, Clone)]
 pub struct Message {
@@ -66,12 +69,12 @@ pub struct Message {
 
  // TODO: Not a fan of this field and we could get rid of it by making TimestampTagger an implicit part of TuneableConsistencyScatter
  // This metadata field is only used for communication between transforms and should not be touched by sinks or sources
- pub meta_timestamp: Option<i64>,
+ pub(crate) meta_timestamp: Option<i64>,
 
- pub codec_state: CodecState,
+ pub(crate) codec_state: CodecState,
 }
 
-/// `from_*` methods for `Message`
+// `from_*` methods for `Message`
 impl Message {
  /// This method should be called when you have have just the raw bytes of a message.
  /// This is expected to be used only by codecs that are decoding a protocol where the length of the message is provided in the header. e.g. cassandra
@@ -110,7 +113,7 @@ impl Message {
  }
 }
 
-/// Methods for interacting with `Message::inner`
+// Methods for interacting with `Message::inner`
 impl Message {
  /// Returns a `&mut Frame` which contains the processed contents of the message.
  /// A transform may choose to modify the contents of the `&mut Frame` in order to modify the message that is sent to the DB.

shotover/src/message_value.rs

@@ -1,3 +1,5 @@
+//! Generic representations of data types that appear in messages
+
 use crate::frame::cassandra::to_cassandra_type;
 use crate::frame::RedisFrame;
 use bigdecimal::BigDecimal;

shotover/src/runner.rs

@@ -1,3 +1,4 @@
+//! Tools for initializing shotover in the final binary.
 use crate::config::topology::Topology;
 use crate::config::Config;
 use crate::observability::LogFilterHttpExporter;

shotover/src/tcp.rs

@@ -1,3 +1,5 @@
+//! Use to establish a TCP connection to a DB in a sink transform
+
 use anyhow::{anyhow, Result};
 use std::time::Duration;
 use tokio::{

shotover/src/tls.rs

@@ -1,3 +1,5 @@
+//! Use to establish a TLS connection to a DB in a sink transform
+
 use crate::tcp;
 use anyhow::{anyhow, bail, Context, Error, Result};
 use rustls::client::{InvalidDnsNameError, ServerCertVerified, ServerCertVerifier, WebPkiVerifier};

shotover/src/transforms/mod.rs

@@ -1,3 +1,5 @@
+//! Various types required for defining a transform
+
 use crate::error::ChainResponse;
 use crate::message::Messages;
 use crate::transforms::cassandra::peers_rewrite::CassandraPeersRewrite;