chore(dev): Wrap vector-config in vector-lib as configurable (#…

…18944)

* chore(dev): Wrap `vector-config` in `vector-lib` as `configurable`

There already exists a `vector_lib::config` export so I figured adding
`vector-config` to that module would make things too confusing. Since most of
`vector-config` has to do with the `configurable` terminology, that seemed a
natural fit for this export.

Note that, unlike the previous wrappers, we are unable to get rid of the import
of `vector-config` into the base package. This is because the macros in
`vector-config-macros` explicitly assume the use of `vector-config` in their
coding.

* Add package name hack to avoid needing to import `vector-config` in the root

* Fixup stray FIXME

* Drop the imports of `vector-config-*` into `vector-lib`

Cargo.toml

@@ -141,9 +141,6 @@ opentelemetry-proto = { path = "lib/opentelemetry-proto", optional = true }
 tracing-limit = { path = "lib/tracing-limit" }
 vector-api-client = { path = "lib/vector-api-client", optional = true }
 vector-buffers = { path = "lib/vector-buffers", default-features = false }
-vector-config = { path = "lib/vector-config" }
-vector-config-common = { path = "lib/vector-config-common" }
-vector-config-macros = { path = "lib/vector-config-macros" }
 vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] }
 vector-stream = { path = "lib/vector-stream" }
 vector-vrl-functions = { path = "lib/vector-vrl/functions" }

lib/vector-config-common/src/lib.rs

@@ -1,3 +1,5 @@
+#![deny(warnings)]
+
 // TODO: `darling` is currently strict about accepting only matching literal types for scalar fields i.e. a `f64` field
 // can only be parsed from a string or float literal, but not an integer literal... and float literals have to be in the
 // form of `1000.0`, not `1000`.
@@ -10,10 +12,35 @@
 // `#[configurable(validation(length(min = 1)))]` to indicate the string cannot be empty, when
 // something like `#[configurable(validation(not_empty)]` is a bit more self-evident, and shorter to boot
 
-#![deny(warnings)]
+use std::sync::OnceLock;
+
 pub mod attributes;
 pub mod constants;
 pub mod human_friendly;
 pub mod num;
 pub mod schema;
 pub mod validation;
+
+/// Generate the package name to reach `vector_config` in the output of the macros. This should be
+/// `vector_lib::configurable` in all packages that can import `vector_lib` to allow for a single
+/// import interface, but it needs to explicitly name `vector_config` in all packages that
+/// themselves import `vector_lib`.
+pub fn configurable_package_name_hack() -> proc_macro2::TokenStream {
+    // `TokenStream2` does not implement `Sync`, so we can't use it directly in `OnceLock`. As such,
+    // this hack needs to recreate the package name token stream each time. We can also not return a
+    // string type, which could be stored in the `OnceLock`, as one of the options is a multi-token
+    // value and the string will always be parsed as a single token.
+    static RUNNING_IN_LIB: OnceLock<bool> = OnceLock::new();
+    // We can't use `env!("CARGO_PKG_NAME")` to be able to create a `const`, as that is evaluated
+    // once when this macro package is built rather than when they are evaluated. This has to be
+    // evaluated in the context of the package in which the macro is being expanded.
+    let running_in_lib = *RUNNING_IN_LIB.get_or_init(|| {
+        let package = std::env::var("CARGO_PKG_NAME").expect("Must be built by cargo");
+        package.starts_with("vector-") || package == "file-source" || package == "codecs"
+    });
+    if running_in_lib {
+        syn::parse_quote! { ::vector_config }
+    } else {
+        syn::parse_quote! { ::vector_lib::configurable }
+    }
+}

lib/vector-config-common/src/validation.rs

@@ -4,6 +4,7 @@ use quote::{quote, ToTokens};
 use syn::{Expr, Lit, Meta};
 
 use crate::{
+    configurable_package_name_hack,
     num::{ERR_NUMERIC_OUT_OF_RANGE, NUMERIC_ENFORCED_LOWER_BOUND, NUMERIC_ENFORCED_UPPER_BOUND},
     schema::{InstanceType, SchemaObject},
 };
@@ -138,18 +139,19 @@ impl Format {
 
 impl ToTokens for Format {
     fn to_tokens(&self, tokens: &mut TokenStream) {
+        let vector_config = configurable_package_name_hack();
         let format_tokens = match self {
-            Format::Date => quote! { ::vector_config::validation::Format::Date },
-            Format::Time => quote! { ::vector_config::validation::Format::Time },
-            Format::DateTime => quote! { ::vector_config::validation::Format::DateTime },
-            Format::Duration => quote! { ::vector_config::validation::Format::Duration },
-            Format::Email => quote! { ::vector_config::validation::Format::Email },
-            Format::Hostname => quote! { ::vector_config::validation::Format::Hostname },
-            Format::Uri => quote! { ::vector_config::validation::Format::Uri },
-            Format::IPv4 => quote! { ::vector_config::validation::Format::IPv4 },
-            Format::IPv6 => quote! { ::vector_config::validation::Format::IPv6 },
-            Format::Uuid => quote! { ::vector_config::validation::Format::Uuid },
-            Format::Regex => quote! { ::vector_config::validation::Format::Regex },
+            Format::Date => quote! { #vector_config::validation::Format::Date },
+            Format::Time => quote! { #vector_config::validation::Format::Time },
+            Format::DateTime => quote! { #vector_config::validation::Format::DateTime },
+            Format::Duration => quote! { #vector_config::validation::Format::Duration },
+            Format::Email => quote! { #vector_config::validation::Format::Email },
+            Format::Hostname => quote! { #vector_config::validation::Format::Hostname },
+            Format::Uri => quote! { #vector_config::validation::Format::Uri },
+            Format::IPv4 => quote! { #vector_config::validation::Format::IPv4 },
+            Format::IPv6 => quote! { #vector_config::validation::Format::IPv6 },
+            Format::Uuid => quote! { #vector_config::validation::Format::Uuid },
+            Format::Regex => quote! { #vector_config::validation::Format::Regex },
         };
 
         tokens.extend(format_tokens);
@@ -292,24 +294,25 @@ impl Validation {
 
 impl ToTokens for Validation {
     fn to_tokens(&self, tokens: &mut TokenStream) {
+        let vector_config = configurable_package_name_hack();
         let validation_tokens = match self {
             Validation::KnownFormat(format) => {
-                quote! { ::vector_config::validation::Validation::KnownFormat(#format) }
+                quote! { #vector_config::validation::Validation::KnownFormat(#format) }
             }
             Validation::Length { minimum, maximum } => {
                 let min_tokens = option_as_token(*minimum);
                 let max_tokens = option_as_token(*maximum);
 
-                quote! { ::vector_config::validation::Validation::Length { minimum: #min_tokens, maximum: #max_tokens } }
+                quote! { #vector_config::validation::Validation::Length { minimum: #min_tokens, maximum: #max_tokens } }
             }
             Validation::Range { minimum, maximum } => {
                 let min_tokens = option_as_token(*minimum);
                 let max_tokens = option_as_token(*maximum);
 
-                quote! { ::vector_config::validation::Validation::Range { minimum: #min_tokens, maximum: #max_tokens } }
+                quote! { #vector_config::validation::Validation::Range { minimum: #min_tokens, maximum: #max_tokens } }
             }
             Validation::Pattern(pattern) => {
-                quote! { ::vector_config::validation::Validation::Pattern(#pattern.to_string()) }
+                quote! { #vector_config::validation::Validation::Pattern(#pattern.to_string()) }
             }
         };
 

lib/vector-config-macros/src/ast/field.rs

@@ -6,7 +6,7 @@ use proc_macro2::{Span, TokenStream};
 use quote::ToTokens;
 use serde_derive_internals::ast as serde_ast;
 use syn::{parse_quote, ExprPath, Ident};
-use vector_config_common::validation::Validation;
+use vector_config_common::{configurable_package_name_hack, validation::Validation};
 
 use super::{
     util::{
@@ -352,7 +352,8 @@ impl Attributes {
             // serialize wrapper, since we know we'll have to do that in a few different places
             // during codegen, so it's cleaner to do it here.
             let field_ty = field.ty;
-            parse_quote! { ::vector_config::ser::Delegated<#field_ty, #virtual_ty> }
+            let vector_config = configurable_package_name_hack();
+            parse_quote! { #vector_config::ser::Delegated<#field_ty, #virtual_ty> }
         });
 
         Ok(self)

lib/vector-config-macros/src/component_name.rs

@@ -3,6 +3,7 @@ use proc_macro::TokenStream;
 
 use quote::quote;
 use syn::{parse_macro_input, spanned::Spanned, Attribute, DeriveInput, Error, LitStr};
+use vector_config_common::configurable_package_name_hack;
 
 use crate::attrs::{self, path_matches};
 
@@ -89,12 +90,13 @@ pub fn derive_component_name_impl(input: TokenStream) -> TokenStream {
 
     // We have a single, valid component name, so let's actually spit out our derive.
     let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
+    let vector_config = configurable_package_name_hack();
     let derived = quote! {
         impl #impl_generics #ident #ty_generics #where_clause {
             pub(super) const NAME: &'static str = #component_name;
         }
 
-        impl #impl_generics ::vector_config::NamedComponent for #ident #ty_generics #where_clause {
+        impl #impl_generics #vector_config::NamedComponent for #ident #ty_generics #where_clause {
             fn get_component_name(&self) -> &'static str {
                 #component_name
             }