From da5250397f959cd809d256c21f2daf230cc64313 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Wed, 1 Nov 2023 14:02:20 -0600 Subject: [PATCH 1/5] chore(core): Add a CLI flag to allow for empty configs This allows for users to set up an empty config that may later be replaced with actual running components. With this option enabled, Vector will not immediately shut down with an empty config, but it will follow the normal shutdown path when components are present. --- src/app.rs | 30 +++++++++++++++++++++--------- src/cli.rs | 7 +++++++ src/config/builder.rs | 6 ++++++ src/config/compiler.rs | 1 + src/config/loading/mod.rs | 3 +++ src/config/mod.rs | 4 ++++ src/config/validation.rs | 12 +++++++----- 7 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/app.rs b/src/app.rs index 259d846ea94eb..5f8c72c49d8a4 100644 --- a/src/app.rs +++ b/src/app.rs @@ -55,7 +55,7 @@ pub struct ApplicationConfig { } pub struct Application { - pub require_healthy: Option, + pub root_opts: RootOpts, pub config: ApplicationConfig, pub signals: SignalPair, pub openssl_providers: Option>, @@ -75,6 +75,7 @@ impl ApplicationConfig { &config_paths, opts.watch_config, opts.require_healthy, + opts.allow_empty_config, graceful_shutdown_duration, signal_handler, ) @@ -219,7 +220,7 @@ impl Application { Ok(( runtime, Self { - require_healthy: opts.root.require_healthy, + root_opts: opts.root, config, signals, openssl_providers, @@ -236,7 +237,7 @@ impl Application { handle.spawn(heartbeat::heartbeat()); let Self { - require_healthy, + root_opts, config, signals, openssl_providers, @@ -247,7 +248,7 @@ impl Application { api_server: config.setup_api(handle), topology: config.topology, config_paths: config.config_paths.clone(), - require_healthy, + require_healthy: root_opts.require_healthy, #[cfg(feature = "enterprise")] enterprise_reporter: config.enterprise, }); @@ -259,6 +260,7 @@ impl Application { signals, topology_controller, openssl_providers, + allow_empty_config: root_opts.allow_empty_config, }) } } @@ -270,6 +272,7 @@ pub struct StartedApplication { pub signals: SignalPair, pub topology_controller: SharedTopologyController, pub openssl_providers: Option>, + pub allow_empty_config: bool, } impl StartedApplication { @@ -285,6 +288,7 @@ impl StartedApplication { topology_controller, openssl_providers, internal_topologies, + allow_empty_config, } = self; let mut graceful_crash = UnboundedReceiverStream::new(graceful_crash_receiver); @@ -293,18 +297,20 @@ impl StartedApplication { let mut signal_rx = signals.receiver; let signal = loop { + let has_sources = !topology_controller.lock().await.topology.config.is_empty(); tokio::select! { signal = signal_rx.recv() => if let Some(signal) = handle_signal( signal, &topology_controller, &config_paths, &mut signal_handler, + allow_empty_config, ).await { break signal; }, // Trigger graceful shutdown if a component crashed, or all sources have ended. error = graceful_crash.next() => break SignalTo::Shutdown(error), - _ = TopologyController::sources_finished(topology_controller.clone()) => { + _ = TopologyController::sources_finished(topology_controller.clone()), if has_sources => { info!("All sources have finished."); break SignalTo::Shutdown(None) } , @@ -327,6 +333,7 @@ async fn handle_signal( topology_controller: &SharedTopologyController, config_paths: &[ConfigPath], signal_handler: &mut SignalHandler, + allow_empty_config: bool, ) -> Option { match signal { Ok(SignalTo::ReloadFromConfigBuilder(config_builder)) => { @@ -349,6 +356,7 @@ async fn handle_signal( let new_config = config::load_from_paths_with_provider_and_secrets( &topology_controller.config_paths, signal_handler, + allow_empty_config, ) .await .map_err(handle_config_errors) @@ -496,6 +504,7 @@ pub async fn load_configs( config_paths: &[ConfigPath], watch_config: bool, require_healthy: Option, + allow_empty_config: bool, graceful_shutdown_duration: Option, signal_handler: &mut SignalHandler, ) -> Result { @@ -520,10 +529,13 @@ pub async fn load_configs( #[cfg(not(feature = "enterprise-tests"))] config::init_log_schema(&config_paths, true).map_err(handle_config_errors)?; - let mut config = - config::load_from_paths_with_provider_and_secrets(&config_paths, signal_handler) - .await - .map_err(handle_config_errors)?; + let mut config = config::load_from_paths_with_provider_and_secrets( + &config_paths, + signal_handler, + allow_empty_config, + ) + .await + .map_err(handle_config_errors)?; config::init_telemetry(config.global.telemetry.clone(), true); diff --git a/src/cli.rs b/src/cli.rs index 54ec9cbb747c4..69e1e7aeaaced 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -216,6 +216,13 @@ pub struct RootOpts { /// default inherits the environment of the Vector process. #[arg(long, env = "VECTOR_OPENSSL_NO_PROBE", default_value = "false")] pub openssl_no_probe: bool, + + /// Allow the configuration to run without any components. This is useful for loading in an + /// empty stub config that will later be replaced with actual components. Note that this is + /// likely not useful without also watching for config file changes as described for + /// `--watch-empty`. + #[arg(long, env = "VECTOR_ALLOW_EMPTY", default_value = "false")] + pub allow_empty_config: bool, } impl RootOpts { diff --git a/src/config/builder.rs b/src/config/builder.rs index e3febfba5843a..f67322036fbd1 100644 --- a/src/config/builder.rs +++ b/src/config/builder.rs @@ -83,6 +83,11 @@ pub struct ConfigBuilder { #[serde(default, skip)] #[doc(hidden)] pub graceful_shutdown_duration: Option, + + /// Allow the configuration to be empty, resulting in a topology with no components. + #[serde(default, skip)] + #[doc(hidden)] + pub allow_empty: bool, } #[cfg(feature = "enterprise")] @@ -232,6 +237,7 @@ impl From for ConfigBuilder { tests, secret, graceful_shutdown_duration, + allow_empty: false, } } } diff --git a/src/config/compiler.rs b/src/config/compiler.rs index cff03862375b9..e5a3988d50386 100644 --- a/src/config/compiler.rs +++ b/src/config/compiler.rs @@ -57,6 +57,7 @@ pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec), Vec< provider: _, secret, graceful_shutdown_duration, + allow_empty: _, } = builder; let graph = match Graph::new(&sources, &transforms, &sinks, schema) { diff --git a/src/config/loading/mod.rs b/src/config/loading/mod.rs index ad5fe33371e79..d398211deee39 100644 --- a/src/config/loading/mod.rs +++ b/src/config/loading/mod.rs @@ -133,6 +133,7 @@ pub fn load_from_paths(config_paths: &[ConfigPath]) -> Result Result> { // Load secret backends first let (mut secrets_backends_loader, secrets_warning) = @@ -149,6 +150,8 @@ pub async fn load_from_paths_with_provider_and_secrets( load_builder_from_paths(config_paths)? }; + builder.allow_empty = allow_empty; + validation::check_provider(&builder)?; signal_handler.clear(); diff --git a/src/config/mod.rs b/src/config/mod.rs index 5386cdcc2d147..4d39f5e1fd804 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -127,6 +127,10 @@ impl Config { Default::default() } + pub fn is_empty(&self) -> bool { + self.sources.is_empty() + } + pub fn sources(&self) -> impl Iterator { self.sources.iter() } diff --git a/src/config/validation.rs b/src/config/validation.rs index 53bbe64b95543..3a3a26a886595 100644 --- a/src/config/validation.rs +++ b/src/config/validation.rs @@ -44,12 +44,14 @@ pub fn check_names<'a, I: Iterator>(names: I) -> Result pub fn check_shape(config: &ConfigBuilder) -> Result<(), Vec> { let mut errors = vec![]; - if config.sources.is_empty() { - errors.push("No sources defined in the config.".to_owned()); - } + if !config.allow_empty { + if config.sources.is_empty() { + errors.push("No sources defined in the config.".to_owned()); + } - if config.sinks.is_empty() { - errors.push("No sinks defined in the config.".to_owned()); + if config.sinks.is_empty() { + errors.push("No sinks defined in the config.".to_owned()); + } } // Helper for below From 19c1fd9df5d245d1cb0fc8f322a22472871e88d0 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Thu, 2 Nov 2023 09:24:30 -0600 Subject: [PATCH 2/5] Add reference docs --- src/cli.rs | 2 +- website/cue/reference/cli.cue | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/cli.rs b/src/cli.rs index 69e1e7aeaaced..8bfb44717a5ed 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -221,7 +221,7 @@ pub struct RootOpts { /// empty stub config that will later be replaced with actual components. Note that this is /// likely not useful without also watching for config file changes as described for /// `--watch-empty`. - #[arg(long, env = "VECTOR_ALLOW_EMPTY", default_value = "false")] + #[arg(long, env = "VECTOR_ALLOW_EMPTY_CONFIG", default_value = "false")] pub allow_empty_config: bool, } diff --git a/website/cue/reference/cli.cue b/website/cue/reference/cli.cue index df5f1d8c8b98a..992864f82bcdb 100644 --- a/website/cue/reference/cli.cue +++ b/website/cue/reference/cli.cue @@ -121,6 +121,10 @@ cli: { description: env_vars.VECTOR_OPENSSL_NO_PROBE.description env_var: "VECTOR_OPENSSL_NO_PROBE" } + "allow-empty-config": { + description: env_vars.VECTOR_ALLOW_EMPTY_CONFIG.description + env_var: "VECTOR_ALLOW_EMPTY_CONFIG" + } } _core_config_options: { @@ -644,6 +648,12 @@ cli: { """ type: bool: default: false } + VECTOR_ALLOW_EMPTY_CONFIG: { + description: """ + Allow the configuration to run without any components. This is useful for loading in an empty stub config that will later be replaced with actual components. Note that this is likely not useful without also watching for config file changes as described for `--watch-empty`. + """ + type: bool: default: false + } } // Helpers From 38058800f1ffebcff4ed791c2808ed5957459638 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Thu, 2 Nov 2023 09:53:53 -0600 Subject: [PATCH 3/5] Fix tabs in cue docs --- website/cue/reference/cli.cue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/cue/reference/cli.cue b/website/cue/reference/cli.cue index 992864f82bcdb..562b948ea1fd4 100644 --- a/website/cue/reference/cli.cue +++ b/website/cue/reference/cli.cue @@ -650,8 +650,8 @@ cli: { } VECTOR_ALLOW_EMPTY_CONFIG: { description: """ - Allow the configuration to run without any components. This is useful for loading in an empty stub config that will later be replaced with actual components. Note that this is likely not useful without also watching for config file changes as described for `--watch-empty`. - """ + Allow the configuration to run without any components. This is useful for loading in an empty stub config that will later be replaced with actual components. Note that this is likely not useful without also watching for config file changes as described for `--watch-empty`. + """ type: bool: default: false } } From 7ac746a22933c8a837c45144c876cd582e35d0a3 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Thu, 2 Nov 2023 11:11:27 -0600 Subject: [PATCH 4/5] Tweak wording --- src/cli.rs | 2 +- website/cue/reference/cli.cue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 8bfb44717a5ed..6854c3d9a36eb 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -219,7 +219,7 @@ pub struct RootOpts { /// Allow the configuration to run without any components. This is useful for loading in an /// empty stub config that will later be replaced with actual components. Note that this is - /// likely not useful without also watching for config file changes as described for + /// likely not useful without also watching for config file changes as described in /// `--watch-empty`. #[arg(long, env = "VECTOR_ALLOW_EMPTY_CONFIG", default_value = "false")] pub allow_empty_config: bool, diff --git a/website/cue/reference/cli.cue b/website/cue/reference/cli.cue index 562b948ea1fd4..2dafd93d636ab 100644 --- a/website/cue/reference/cli.cue +++ b/website/cue/reference/cli.cue @@ -650,7 +650,7 @@ cli: { } VECTOR_ALLOW_EMPTY_CONFIG: { description: """ - Allow the configuration to run without any components. This is useful for loading in an empty stub config that will later be replaced with actual components. Note that this is likely not useful without also watching for config file changes as described for `--watch-empty`. + Allow the configuration to run without any components. This is useful for loading in an empty stub config that will later be replaced with actual components. Note that this is likely not useful without also watching for config file changes as described in `--watch-empty`. """ type: bool: default: false } From 77c1c2e02c7716bcb50798acfe19acd482c297dc Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Thu, 2 Nov 2023 11:12:41 -0600 Subject: [PATCH 5/5] Fix option typo --- src/cli.rs | 2 +- website/cue/reference/cli.cue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 6854c3d9a36eb..0d1667c615f1b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -220,7 +220,7 @@ pub struct RootOpts { /// Allow the configuration to run without any components. This is useful for loading in an /// empty stub config that will later be replaced with actual components. Note that this is /// likely not useful without also watching for config file changes as described in - /// `--watch-empty`. + /// `--watch-config`. #[arg(long, env = "VECTOR_ALLOW_EMPTY_CONFIG", default_value = "false")] pub allow_empty_config: bool, } diff --git a/website/cue/reference/cli.cue b/website/cue/reference/cli.cue index 2dafd93d636ab..ee07ca2ce91be 100644 --- a/website/cue/reference/cli.cue +++ b/website/cue/reference/cli.cue @@ -650,7 +650,7 @@ cli: { } VECTOR_ALLOW_EMPTY_CONFIG: { description: """ - Allow the configuration to run without any components. This is useful for loading in an empty stub config that will later be replaced with actual components. Note that this is likely not useful without also watching for config file changes as described in `--watch-empty`. + Allow the configuration to run without any components. This is useful for loading in an empty stub config that will later be replaced with actual components. Note that this is likely not useful without also watching for config file changes as described in `--watch-config`. """ type: bool: default: false }