feat: override config with cli opts (#7613)

Define a proc macro `OverrideConfig` which will override fields in `RwConfig`. It supports overriding any field in the config file. Its usage is as bellow:

```rust
pub struct ComputeNodeOpts {
// Other items ...

#[clap(flatten)]
override_config: OverrideConfigOpts,
}

/// Command-line arguments for compute-node that overrides the config file.
#[derive(Parser, Clone, Debug, OverrideConfig)]
struct OverrideConfigOpts {
#[clap(long)]
#[override(path = storage.state_store)]
pub state_store: Option<String>,

/// Enable reporting tracing information to jaeger.
#[clap(parse(from_flag), long)]
#[override(path = streaming.enable_jaeger_tracing)]
pub enable_jaeger_tracing: Flag,
}

fn compute_node_serve(opts: ComputeNodeOpts) {
// `override_config` should not be usable afterwards
let config = load_config(&opts.config_path, Some(opts.override_config));
}
```

I plan to keep only the addresses, hardware-related items and credential-related items in CLI. Thus more options are available in the config file. Please check the release notes.

This PR keeps backward compatibility, as the newly added fields in the config file can still be overridden from CLI.

I did not use serfig because it [does not support overriding with default values](Xuanwo/serfig#23 (comment)). The user might not do that, but as risedev users, we might accidentally have modified risingwave.toml and forgot to change it back, which conflicts with the values generated by risedev and causing confusing behaviours 🥵

Approved-By: fuyufjh
Approved-By: xxchan

Co-Authored-By: Gun9niR <gun9nir.guo@gmail.com>
Co-Authored-By: Zhidong Guo <52783948+Gun9niR@users.noreply.github.com>

src/cmd_all/src/playground.rs

@@ -18,7 +18,6 @@ use std::sync::LazyLock;
 
 use anyhow::Result;
 use clap::StructOpt;
-use risingwave_common::config::load_config;
 use tempfile::TempPath;
 use tokio::signal;
 
@@ -154,9 +153,6 @@ pub async fn playground() -> Result<()> {
                 opts.insert(0, "meta-node".into());
                 tracing::info!("starting meta-node thread with cli args: {:?}", opts);
                 let opts = risingwave_meta::MetaNodeOpts::parse_from(opts);
-
-                let _config = load_config(&opts.config_path);
-
                 tracing::info!("opts: {:#?}", opts);
                 let _meta_handle = tokio::spawn(async move {
                     risingwave_meta::start(opts).await;

src/common/Cargo.toml

@@ -19,6 +19,7 @@ byteorder = "1"
 bytes = "1"
 chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
 chrono-tz = { version = "0.7", features = ["case-insensitive"] }
+clap = { version = "3", features = ["derive"] }
 comfy-table = "6"
 crc32fast = "1"
 derivative = "2"

src/common/proc_macro/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "risingwave_common_proc_macro"
+version = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro-error = "1.0"
+quote = "1"
+proc-macro2 = { version = "1", default-features = false }
+syn = "1"
+bae = "0.1.7"
+
+[target.'cfg(not(madsim))'.dependencies]
+workspace-hack = { path = "../../workspace-hack" }

src/common/proc_macro/src/config.rs

@@ -0,0 +1,64 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use bae::FromAttributes;
+use proc_macro2::TokenStream;
+use proc_macro_error::ResultExt;
+use quote::quote;
+use syn::DeriveInput;
+
+#[derive(FromAttributes)]
+pub struct OverrideOpts {
+    pub path: Option<syn::Expr>,
+    pub optional_in_config: Option<()>,
+}
+
+#[cfg_attr(coverage, no_coverage)]
+pub fn produce_override_config(input: DeriveInput) -> TokenStream {
+    let struct_ident = input.ident;
+    let mut override_stmts = Vec::new();
+
+    if let syn::Data::Struct(syn::DataStruct { fields, .. }) = input.data {
+        for field in fields {
+            let override_opts = OverrideOpts::from_attributes(&field.attrs)
+                .expect_or_abort("Failed to parse `override_opts` attribute");
+            let path = override_opts.path.expect("`path` must exist");
+            let field_ident = field.ident;
+
+            let override_stmt = if override_opts.optional_in_config.is_some() {
+                quote! {
+                    if self.#field_ident.is_some() {
+                        config.#path = self.#field_ident;
+                    }
+                }
+            } else {
+                quote! {
+                    if let Some(v) = self.#field_ident {
+                        config.#path = v;
+                    }
+                }
+            };
+
+            override_stmts.push(override_stmt);
+        }
+    }
+
+    quote! {
+        impl risingwave_common::config::OverrideConfig for #struct_ident {
+            fn r#override(self, config: &mut risingwave_common::config::RwConfig) {
+                #(#override_stmts)*
+            }
+        }
+    }
+}

src/common/proc_macro/src/lib.rs

@@ -0,0 +1,57 @@
+// Copyright 2023 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![cfg_attr(coverage, feature(no_coverage))]
+
+use proc_macro_error::proc_macro_error;
+use syn::parse_macro_input;
+
+mod config;
+
+/// Sections in the configuration file can use `#[derive(OverrideConfig)]` to generate the
+/// implementation of overwriting configs from the file.
+///
+/// In the struct definition, use #[override_opts(path = ...)] on a field to indicate the field in
+/// `RwConfig` to override.
+///
+/// An example:
+///
+/// ```ignore
+/// #[derive(OverrideConfig)]
+/// struct Opts {
+///     #[override_opts(path = meta.listen_addr)]
+///     listen_addr: Option<String>,
+/// }
+/// ```
+///
+/// will generate
+///
+/// impl OverrideConfig for Opts {
+///     fn r#override(self, config: &mut RwConfig) {
+///         if let Some(v) = self.required_str {
+///             config.meta.listen_addr = v;
+///         }
+///     }
+/// }
+/// ```
+#[cfg_attr(coverage, no_coverage)]
+#[proc_macro_derive(OverrideConfig, attributes(override_opts))]
+#[proc_macro_error]
+pub fn override_config(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+    let input = parse_macro_input!(input);
+
+    let gen = config::produce_override_config(input);
+
+    gen.into()
+}