chore(dev): Add wrapper for codecs to vector-lib (#18959)

chore(dev): Add wrapper for `codecs` under `vector-lib`

Cargo.toml

@@ -129,7 +129,6 @@ vrl.workspace = true
 pin-project.workspace = true
 
 # Internal libs
-codecs = { path = "lib/codecs", default-features = false }
 dnsmsg-parser = { path = "lib/dnsmsg-parser", optional = true }
 enrichment = { path = "lib/enrichment" }
 fakedata = { path = "lib/fakedata", optional = true }
@@ -468,7 +467,7 @@ enrichment-tables = ["enrichment-tables-geoip"]
 enrichment-tables-geoip = ["dep:maxminddb"]
 
 # Codecs
-codecs-syslog = ["codecs/syslog"]
+codecs-syslog = ["vector-lib/syslog"]
 
 # Sources
 sources = ["sources-logs", "sources-metrics"]

lib/vector-lib/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 publish = false
 
 [dependencies]
+codecs = { path = "../codecs", default-features = false }
 vector-common = { path = "../vector-common" }
 vector-config = { path = "../vector-config" }
 vector-core = { path = "../vector-core", default-features = false }
@@ -14,5 +15,6 @@ vector-stream = { path = "../vector-stream" }
 [features]
 api = ["vector-core/api"]
 lua = ["vector-core/lua"]
+syslog = ["codecs/syslog"]
 test = ["vector-core/test"]
 vrl = ["vector-core/vrl"]

lib/vector-lib/src/lib.rs

@@ -1,3 +1,4 @@
+pub use codecs;
 pub use vector_common::{
     assert_event_data_eq, btreemap, byte_size_of, byte_size_of::ByteSizeOf, conversion,
     encode_logfmt, finalization, finalizer, impl_event_data_eq, internal_event, json_size,

src/codecs/decoding/config.rs

@@ -1,5 +1,5 @@
-use codecs::decoding::{DeserializerConfig, FramingConfig};
 use serde::{Deserialize, Serialize};
+use vector_lib::codecs::decoding::{DeserializerConfig, FramingConfig};
 use vector_lib::config::LogNamespace;
 
 use crate::codecs::Decoder;

src/codecs/decoding/decoder.rs

@@ -1,9 +1,9 @@
 use bytes::{Bytes, BytesMut};
-use codecs::decoding::{
+use smallvec::SmallVec;
+use vector_lib::codecs::decoding::{
     format::Deserializer as _, BoxedFramingError, BytesDeserializer, Deserializer, Error, Framer,
     NewlineDelimitedDecoder,
 };
-use smallvec::SmallVec;
 use vector_lib::config::LogNamespace;
 
 use crate::{
@@ -103,12 +103,12 @@ impl tokio_util::codec::Decoder for Decoder {
 mod tests {
     use super::Decoder;
     use bytes::Bytes;
-    use codecs::{
+    use futures::{stream, StreamExt};
+    use tokio_util::{codec::FramedRead, io::StreamReader};
+    use vector_lib::codecs::{
         decoding::{Deserializer, Framer},
         JsonDeserializer, NewlineDelimitedDecoder, StreamDecodingError,
     };
-    use futures::{stream, StreamExt};
-    use tokio_util::{codec::FramedRead, io::StreamReader};
     use vrl::value::Value;
 
     #[tokio::test]

src/codecs/encoding/config.rs

@@ -1,5 +1,5 @@
 use crate::codecs::Transformer;
-use codecs::{
+use vector_lib::codecs::{
     encoding::{Framer, FramingConfig, Serializer, SerializerConfig},
     CharacterDelimitedEncoder, LengthDelimitedEncoder, NewlineDelimitedEncoder,
 };

src/codecs/encoding/encoder.rs

@@ -1,9 +1,9 @@
 use bytes::BytesMut;
-use codecs::{
+use tokio_util::codec::Encoder as _;
+use vector_lib::codecs::{
     encoding::{Error, Framer, Serializer},
     CharacterDelimitedEncoder, NewlineDelimitedEncoder, TextSerializerConfig,
 };
-use tokio_util::codec::Encoder as _;
 
 use crate::{
     event::Event,
@@ -184,9 +184,9 @@ impl tokio_util::codec::Encoder<Event> for Encoder<()> {
 #[cfg(test)]
 mod tests {
     use bytes::BufMut;
-    use codecs::encoding::BoxedFramingError;
     use futures_util::{SinkExt, StreamExt};
     use tokio_util::codec::FramedWrite;
+    use vector_lib::codecs::encoding::BoxedFramingError;
     use vector_lib::event::LogEvent;
 
     use super::*;

src/components/validation/resources/event.rs

@@ -4,7 +4,7 @@ use snafu::Snafu;
 use tokio_util::codec::Encoder as _;
 
 use crate::codecs::Encoder;
-use codecs::{
+use vector_lib::codecs::{
     encoding, JsonSerializer, LengthDelimitedEncoder, LogfmtSerializer, MetricTagValues,
     NewlineDelimitedEncoder,
 };

src/components/validation/resources/mod.rs

@@ -1,14 +1,14 @@
 mod event;
 mod http;
 
-use codecs::{
+use tokio::sync::mpsc;
+use vector_lib::codecs::{
     decoding::{self, DeserializerConfig},
     encoding::{
         self, Framer, FramingConfig, JsonSerializerConfig, SerializerConfig, TextSerializerConfig,
     },
     BytesEncoder,
 };
-use tokio::sync::mpsc;
 use vector_lib::{config::DataType, event::Event};
 
 use crate::codecs::{Decoder, DecodingConfig, Encoder, EncodingConfig, EncodingConfigWithFraming};
@@ -143,8 +143,8 @@ fn deserializer_config_to_serializer(config: &DeserializerConfig) -> encoding::S
         DeserializerConfig::Bytes => SerializerConfig::Text(TextSerializerConfig::default()),
         DeserializerConfig::Json { .. } => SerializerConfig::Json(JsonSerializerConfig::default()),
         DeserializerConfig::Protobuf(config) => {
-            SerializerConfig::Protobuf(codecs::encoding::ProtobufSerializerConfig {
-                protobuf: codecs::encoding::ProtobufSerializerOptions {
+            SerializerConfig::Protobuf(vector_lib::codecs::encoding::ProtobufSerializerConfig {
+                protobuf: vector_lib::codecs::encoding::ProtobufSerializerOptions {
                     desc_file: config.protobuf.desc_file.clone(),
                     message_type: config.protobuf.message_type.clone(),
                 },
@@ -197,8 +197,8 @@ fn serializer_config_to_deserializer(
         SerializerConfig::Native => DeserializerConfig::Native,
         SerializerConfig::NativeJson => DeserializerConfig::NativeJson(Default::default()),
         SerializerConfig::Protobuf(config) => {
-            DeserializerConfig::Protobuf(codecs::decoding::ProtobufDeserializerConfig {
-                protobuf: codecs::decoding::ProtobufDeserializerOptions {
+            DeserializerConfig::Protobuf(vector_lib::codecs::decoding::ProtobufDeserializerConfig {
+                protobuf: vector_lib::codecs::decoding::ProtobufDeserializerOptions {
                     desc_file: config.protobuf.desc_file.clone(),
                     message_type: config.protobuf.message_type.clone(),
                 },

src/components/validation/runner/mod.rs

@@ -18,7 +18,7 @@ use tokio::{
 };
 use tokio_util::codec::Encoder as _;
 
-use codecs::encoding;
+use vector_lib::codecs::encoding;
 use vector_lib::{event::Event, EstimatedJsonEncodedSizeOf};
 
 use crate::{

src/internal_events/codecs.rs

@@ -52,7 +52,7 @@ impl<'a> InternalEvent for DecoderDeserializeError<'a> {
 
 #[derive(Debug)]
 pub struct EncoderFramingError<'a> {
-    pub error: &'a codecs::encoding::BoxedFramingError,
+    pub error: &'a vector_lib::codecs::encoding::BoxedFramingError,
 }
 
 impl<'a> InternalEvent for EncoderFramingError<'a> {

src/internal_events/journald.rs

@@ -1,5 +1,5 @@
-use codecs::decoding::BoxedFramingError;
 use metrics::counter;
+use vector_lib::codecs::decoding::BoxedFramingError;
 use vector_lib::internal_event::InternalEvent;
 
 use vector_lib::internal_event::{error_stage, error_type};

src/serde.rs

@@ -1,10 +1,10 @@
 #![allow(missing_docs)]
-use codecs::{
+use indexmap::map::IndexMap;
+use serde::{Deserialize, Serialize};
+use vector_lib::codecs::{
     decoding::{DeserializerConfig, FramingConfig},
     BytesDecoderConfig, BytesDeserializerConfig,
 };
-use indexmap::map::IndexMap;
-use serde::{Deserialize, Serialize};
 use vector_lib::configurable::configurable_component;
 pub use vector_lib::serde::{bool_or_struct, skip_serializing_if_default};
 

src/sinks/amqp/config.rs

@@ -1,8 +1,8 @@
 //! Configuration functionality for the `AMQP` sink.
 use crate::{amqp::AmqpConfig, sinks::prelude::*};
-use codecs::TextSerializerConfig;
 use lapin::{types::ShortString, BasicProperties};
 use std::sync::Arc;
+use vector_lib::codecs::TextSerializerConfig;
 
 use super::sink::AmqpSink;
 

src/sinks/aws_cloudwatch_logs/config.rs

@@ -1,8 +1,8 @@
 use aws_sdk_cloudwatchlogs::Client as CloudwatchLogsClient;
 use aws_smithy_types::retry::RetryConfig;
-use codecs::JsonSerializerConfig;
 use futures::FutureExt;
 use tower::ServiceBuilder;
+use vector_lib::codecs::JsonSerializerConfig;
 use vector_lib::configurable::configurable_component;
 use vector_lib::schema;
 use vrl::value::Kind;

src/sinks/aws_cloudwatch_logs/integration_tests.rs

@@ -3,9 +3,9 @@ use std::convert::TryFrom;
 use aws_sdk_cloudwatchlogs::Client as CloudwatchLogsClient;
 use aws_sdk_cloudwatchlogs::Region;
 use chrono::Duration;
-use codecs::TextSerializerConfig;
 use futures::{stream, StreamExt};
 use similar_asserts::assert_eq;
+use vector_lib::codecs::TextSerializerConfig;
 
 use super::*;
 use crate::aws::create_client;

src/sinks/aws_kinesis/firehose/integration_tests.rs

@@ -3,10 +3,10 @@
 
 use aws_sdk_elasticsearch::Client as EsClient;
 use aws_sdk_firehose::model::ElasticsearchDestinationConfiguration;
-use codecs::JsonSerializerConfig;
 use futures::TryFutureExt;
 use serde_json::{json, Value};
 use tokio::time::{sleep, Duration};
+use vector_lib::codecs::JsonSerializerConfig;
 
 use super::{config::KinesisFirehoseClientBuilder, *};
 use crate::{

src/sinks/aws_kinesis/firehose/tests.rs

@@ -1,6 +1,6 @@
 #![cfg(test)]
 
-use codecs::JsonSerializerConfig;
+use vector_lib::codecs::JsonSerializerConfig;
 
 use super::*;
 use crate::{

src/sinks/aws_kinesis/streams/integration_tests.rs

@@ -5,8 +5,8 @@ use aws_sdk_kinesis::{
     model::{Record, ShardIteratorType},
     types::DateTime,
 };
-use codecs::TextSerializerConfig;
 use tokio::time::{sleep, Duration};
+use vector_lib::codecs::TextSerializerConfig;
 
 use super::{config::KinesisClientBuilder, *};
 use crate::{

src/sinks/aws_s3/config.rs

@@ -1,11 +1,11 @@
 use std::convert::TryInto;
 
 use aws_sdk_s3::Client as S3Client;
-use codecs::{
+use tower::ServiceBuilder;
+use vector_lib::codecs::{
     encoding::{Framer, FramingConfig},
     TextSerializerConfig,
 };
-use tower::ServiceBuilder;
 use vector_lib::configurable::configurable_component;
 use vector_lib::sink::VectorSink;
 

src/sinks/aws_s3/integration_tests.rs

@@ -16,11 +16,11 @@ use aws_sdk_s3::{
     Client as S3Client,
 };
 use bytes::Buf;
-use codecs::{encoding::FramingConfig, TextSerializerConfig};
 use flate2::read::MultiGzDecoder;
 use futures::{stream, Stream};
 use similar_asserts::assert_eq;
 use tokio_stream::StreamExt;
+use vector_lib::codecs::{encoding::FramingConfig, TextSerializerConfig};
 use vector_lib::{
     config::proxy::ProxyConfig,
     event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event, EventArray, LogEvent},

src/sinks/aws_s3/sink.rs

@@ -2,8 +2,8 @@ use std::io;
 
 use bytes::Bytes;
 use chrono::Utc;
-use codecs::encoding::Framer;
 use uuid::Uuid;
+use vector_lib::codecs::encoding::Framer;
 use vector_lib::event::Finalizable;
 use vector_lib::request_metadata::RequestMetadata;
 

src/sinks/aws_s_s/sns/integration_tests.rs

@@ -3,8 +3,8 @@ use std::collections::HashMap;
 use aws_sdk_sns::{Client as SnsClient, Region};
 use aws_sdk_sqs::model::QueueAttributeName;
 use aws_sdk_sqs::Client as SqsClient;
-use codecs::TextSerializerConfig;
 use tokio::time::{sleep, Duration};
+use vector_lib::codecs::TextSerializerConfig;
 
 use super::{
     config::SnsClientBuilder,

src/sinks/aws_s_s/sqs/integration_tests.rs

@@ -1,8 +1,8 @@
 use std::collections::HashMap;
 
 use aws_sdk_sqs::{model::QueueAttributeName, Client as SqsClient, Region};
-use codecs::TextSerializerConfig;
 use tokio::time::{sleep, Duration};
+use vector_lib::codecs::TextSerializerConfig;
 
 use crate::config::{SinkConfig, SinkContext};
 use crate::sinks::aws_s_s::sqs::config::{healthcheck, SqsSinkConfig};

src/sinks/azure_blob/config.rs

@@ -1,8 +1,8 @@
 use std::sync::Arc;
 
 use azure_storage_blobs::prelude::*;
-use codecs::{encoding::Framer, JsonSerializerConfig, NewlineDelimitedEncoderConfig};
 use tower::ServiceBuilder;
+use vector_lib::codecs::{encoding::Framer, JsonSerializerConfig, NewlineDelimitedEncoderConfig};
 use vector_lib::configurable::configurable_component;
 use vector_lib::sensitive_string::SensitiveString;
 

src/sinks/azure_blob/integration_tests.rs

@@ -6,13 +6,13 @@ use std::{
 use azure_core::{error::HttpError, prelude::Range};
 use azure_storage_blobs::prelude::*;
 use bytes::{Buf, BytesMut};
-use codecs::{
-    encoding::FramingConfig, JsonSerializerConfig, NewlineDelimitedEncoderConfig,
-    TextSerializerConfig,
-};
 use flate2::read::GzDecoder;
 use futures::{stream, Stream, StreamExt};
 use http::StatusCode;
+use vector_lib::codecs::{
+    encoding::FramingConfig, JsonSerializerConfig, NewlineDelimitedEncoderConfig,
+    TextSerializerConfig,
+};
 use vector_lib::ByteSizeOf;
 
 use super::config::AzureBlobSinkConfig;

src/sinks/azure_blob/request_builder.rs

@@ -1,7 +1,7 @@
 use bytes::Bytes;
 use chrono::Utc;
-use codecs::encoding::Framer;
 use uuid::Uuid;
+use vector_lib::codecs::encoding::Framer;
 use vector_lib::request_metadata::RequestMetadata;
 use vector_lib::EstimatedJsonEncodedSizeOf;
 

src/sinks/azure_blob/test.rs

@@ -1,6 +1,6 @@
 use bytes::Bytes;
 use chrono::Utc;
-use codecs::{
+use vector_lib::codecs::{
     encoding::{Framer, FramingConfig},
     NewlineDelimitedEncoder, TextSerializerConfig,
 };

src/sinks/azure_monitor_logs/sink.rs

@@ -1,8 +1,8 @@
 use std::{fmt::Debug, io};
 
 use bytes::Bytes;
-use codecs::{encoding::Framer, CharacterDelimitedEncoder, JsonSerializerConfig};
 use lookup::{OwnedValuePath, PathPrefix};
+use vector_lib::codecs::{encoding::Framer, CharacterDelimitedEncoder, JsonSerializerConfig};
 
 use crate::sinks::prelude::*;
 

src/sinks/clickhouse/sink.rs

@@ -1,5 +1,5 @@
 use bytes::Bytes;
-use codecs::{encoding::Framer, JsonSerializerConfig, NewlineDelimitedEncoderConfig};
+use vector_lib::codecs::{encoding::Framer, JsonSerializerConfig, NewlineDelimitedEncoderConfig};
 
 use super::service::{ClickhouseRequest, ClickhouseRetryLogic, ClickhouseService};
 use crate::sinks::prelude::*;

src/sinks/console/config.rs

@@ -1,9 +1,9 @@
-use codecs::{
+use futures::{future, FutureExt};
+use tokio::io;
+use vector_lib::codecs::{
     encoding::{Framer, FramingConfig},
     JsonSerializerConfig,
 };
-use futures::{future, FutureExt};
-use tokio::io;
 use vector_lib::configurable::configurable_component;
 
 use crate::{