From a06c71102867af5e4526e445f9ba8f4506382a30 Mon Sep 17 00:00:00 2001 From: Daniel599 Date: Fri, 28 Jul 2023 20:59:05 +0300 Subject: [PATCH] feat(codecs): add support for protobuf decoding (#18019) * feat(codecs): add support for protobuf decoding, WIP still have some TODO to resolve * feat(codecs): add support for protobuf. code-review fixes: handle unwraps and fix support for empty buffer as a message, allowed in protobuf. * feat(codecs): add support for protobuf. code-review fixes: use `kind::any()` instead of `kind::json()`. use `unimplemented!()` instead of `todo!()`. in tests, add checks for List and Map. in `ProtobufDeserializer::new`, refactor out creation of MessageDescriptor. run `cargo fmt`. * feat(codecs): add support for protobuf. code-review fixes: apply suggested refactor to `to_vrl`, it's slightly slower, might improve in following PR. * clippy fixes and minor refactoring * address Bruce's comments * update test code to use new log schema interface * generate docs --------- Co-authored-by: Pavlos Rontidis --- Cargo.lock | 4 + lib/codecs/Cargo.toml | 1 + lib/codecs/src/decoding/format/mod.rs | 2 + lib/codecs/src/decoding/format/protobuf.rs | 353 ++++++++++++++++++ lib/codecs/src/decoding/mod.rs | 16 +- .../data/decoding/protobuf/person_someone.pb | 3 + .../data/decoding/protobuf/person_someone.txt | 2 + .../data/decoding/protobuf/person_someone3.pb | Bin 0 -> 35 bytes .../decoding/protobuf/person_someone3.txt | 2 + .../data/decoding/protobuf/test_protobuf.desc | Bin 0 -> 1183 bytes .../decoding/protobuf/test_protobuf.proto | 26 ++ .../decoding/protobuf/test_protobuf3.desc | Bin 0 -> 1429 bytes .../decoding/protobuf/test_protobuf3.proto | 27 ++ src/components/validation/resources/mod.rs | 1 + .../components/sources/base/amqp.cue | 17 + .../sources/base/aws_kinesis_firehose.cue | 17 + .../components/sources/base/aws_s3.cue | 17 + .../components/sources/base/aws_sqs.cue | 17 + .../components/sources/base/datadog_agent.cue | 17 + .../components/sources/base/demo_logs.cue | 17 + .../components/sources/base/exec.cue | 17 + .../sources/base/file_descriptor.cue | 17 + .../components/sources/base/gcp_pubsub.cue | 17 + .../components/sources/base/heroku_logs.cue | 17 + .../components/sources/base/http.cue | 17 + .../components/sources/base/http_client.cue | 17 + .../components/sources/base/http_server.cue | 17 + .../components/sources/base/kafka.cue | 17 + .../components/sources/base/nats.cue | 17 + .../components/sources/base/redis.cue | 17 + .../components/sources/base/socket.cue | 17 + .../components/sources/base/stdin.cue | 17 + 32 files changed, 742 insertions(+), 1 deletion(-) create mode 100644 lib/codecs/src/decoding/format/protobuf.rs create mode 100644 lib/codecs/tests/data/decoding/protobuf/person_someone.pb create mode 100644 lib/codecs/tests/data/decoding/protobuf/person_someone.txt create mode 100644 lib/codecs/tests/data/decoding/protobuf/person_someone3.pb create mode 100644 lib/codecs/tests/data/decoding/protobuf/person_someone3.txt create mode 100644 lib/codecs/tests/data/decoding/protobuf/test_protobuf.desc create mode 100644 lib/codecs/tests/data/decoding/protobuf/test_protobuf.proto create mode 100644 lib/codecs/tests/data/decoding/protobuf/test_protobuf3.desc create mode 100644 lib/codecs/tests/data/decoding/protobuf/test_protobuf3.proto diff --git a/Cargo.lock b/Cargo.lock index 9ed5ac462caa9..a5f635dafc5cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1968,6 +1968,7 @@ dependencies = [ "once_cell", "ordered-float 3.7.0", "prost", + "prost-reflect", "regex", "serde", "serde_json", @@ -6402,9 +6403,12 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "000e1e05ebf7b26e1eba298e66fe4eee6eb19c567d0ffb35e0dd34231cdac4c8" dependencies = [ + "base64 0.21.2", "once_cell", "prost", "prost-types", + "serde", + "serde-value", ] [[package]] diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 0e34db9025356..b987532c441dd 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -17,6 +17,7 @@ memchr = { version = "2", default-features = false } once_cell = { version = "1.18", default-features = false } ordered-float = { version = "3.7.0", default-features = false } prost = { version = "0.11.8", default-features = false, features = ["std"] } +prost-reflect = { version = "0.11", default-features = false, features = ["serde"] } regex = { version = "1.9.1", default-features = false, features = ["std", "perf"] } serde = { version = "1", default-features = false, features = ["derive"] } serde_json = { version = "1", default-features = false } diff --git a/lib/codecs/src/decoding/format/mod.rs b/lib/codecs/src/decoding/format/mod.rs index c0ab2ea1b5924..5172705448dc2 100644 --- a/lib/codecs/src/decoding/format/mod.rs +++ b/lib/codecs/src/decoding/format/mod.rs @@ -8,6 +8,7 @@ mod gelf; mod json; mod native; mod native_json; +mod protobuf; #[cfg(feature = "syslog")] mod syslog; @@ -19,6 +20,7 @@ pub use native::{NativeDeserializer, NativeDeserializerConfig}; pub use native_json::{ NativeJsonDeserializer, NativeJsonDeserializerConfig, NativeJsonDeserializerOptions, }; +pub use protobuf::{ProtobufDeserializer, ProtobufDeserializerConfig}; use smallvec::SmallVec; #[cfg(feature = "syslog")] pub use syslog::{SyslogDeserializer, SyslogDeserializerConfig, SyslogDeserializerOptions}; diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs new file mode 100644 index 0000000000000..c2b14310a17dc --- /dev/null +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -0,0 +1,353 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::PathBuf; + +use bytes::Bytes; +use chrono::Utc; +use ordered_float::NotNan; +use prost_reflect::{DescriptorPool, DynamicMessage, MessageDescriptor, ReflectMessage}; +use smallvec::{smallvec, SmallVec}; +use vector_config::configurable_component; +use vector_core::event::LogEvent; +use vector_core::{ + config::{log_schema, DataType, LogNamespace}, + event::Event, + schema, +}; +use vrl::value::Kind; + +use super::Deserializer; + +/// Config used to build a `ProtobufDeserializer`. +#[configurable_component] +#[derive(Debug, Clone, Default)] +pub struct ProtobufDeserializerConfig { + /// Path to desc file + desc_file: PathBuf, + + /// message type. e.g package.message + message_type: String, +} + +impl ProtobufDeserializerConfig { + /// Build the `ProtobufDeserializer` from this configuration. + pub fn build(&self) -> ProtobufDeserializer { + // TODO return a Result instead. + ProtobufDeserializer::try_from(self).unwrap() + } + + /// Return the type of event build by this deserializer. + pub fn output_type(&self) -> DataType { + DataType::Log + } + + /// The schema produced by the deserializer. + pub fn schema_definition(&self, log_namespace: LogNamespace) -> schema::Definition { + match log_namespace { + LogNamespace::Legacy => { + let mut definition = + schema::Definition::empty_legacy_namespace().unknown_fields(Kind::any()); + + if let Some(timestamp_key) = log_schema().timestamp_key() { + definition = definition.try_with_field( + timestamp_key, + // The protobuf decoder will try to insert a new `timestamp`-type value into the + // "timestamp_key" field, but only if that field doesn't already exist. + Kind::any().or_timestamp(), + Some("timestamp"), + ); + } + definition + } + LogNamespace::Vector => { + schema::Definition::new_with_default_metadata(Kind::any(), [log_namespace]) + } + } + } +} + +/// Deserializer that builds `Event`s from a byte frame containing protobuf. +#[derive(Debug, Clone)] +pub struct ProtobufDeserializer { + message_descriptor: MessageDescriptor, +} + +impl ProtobufDeserializer { + /// Creates a new `ProtobufDeserializer`. + pub fn new(message_descriptor: MessageDescriptor) -> Self { + Self { message_descriptor } + } + + fn get_message_descriptor( + desc_file: &PathBuf, + message_type: String, + ) -> vector_common::Result { + let b = fs::read(desc_file) + .map_err(|e| format!("Failed to open protobuf desc file '{desc_file:?}': {e}",))?; + let pool = DescriptorPool::decode(b.as_slice()) + .map_err(|e| format!("Failed to parse protobuf desc file '{desc_file:?}': {e}"))?; + Ok(pool.get_message_by_name(&message_type).unwrap_or_else(|| { + panic!("The message type '{message_type}' could not be found in '{desc_file:?}'") + })) + } +} + +impl Deserializer for ProtobufDeserializer { + fn parse( + &self, + bytes: Bytes, + log_namespace: LogNamespace, + ) -> vector_common::Result> { + let dynamic_message = DynamicMessage::decode(self.message_descriptor.clone(), bytes) + .map_err(|error| format!("Error parsing protobuf: {:?}", error))?; + + let proto_vrl = to_vrl(&prost_reflect::Value::Message(dynamic_message), None)?; + let mut event = Event::Log(LogEvent::from(proto_vrl)); + let event = match log_namespace { + LogNamespace::Vector => event, + LogNamespace::Legacy => { + let timestamp = Utc::now(); + if let Some(timestamp_key) = log_schema().timestamp_key_target_path() { + let log = event.as_mut_log(); + if !log.contains(timestamp_key) { + log.insert(timestamp_key, timestamp); + } + } + event + } + }; + + Ok(smallvec![event]) + } +} + +impl TryFrom<&ProtobufDeserializerConfig> for ProtobufDeserializer { + type Error = vector_common::Error; + fn try_from(config: &ProtobufDeserializerConfig) -> vector_common::Result { + let message_descriptor = ProtobufDeserializer::get_message_descriptor( + &config.desc_file, + config.message_type.clone(), + )?; + Ok(Self::new(message_descriptor)) + } +} + +fn to_vrl( + prost_reflect_value: &prost_reflect::Value, + field_descriptor: Option<&prost_reflect::FieldDescriptor>, +) -> vector_common::Result { + let vrl_value = match prost_reflect_value { + prost_reflect::Value::Bool(v) => vrl::value::Value::from(*v), + prost_reflect::Value::I32(v) => vrl::value::Value::from(*v), + prost_reflect::Value::I64(v) => vrl::value::Value::from(*v), + prost_reflect::Value::U32(v) => vrl::value::Value::from(*v), + prost_reflect::Value::U64(v) => vrl::value::Value::from(*v), + prost_reflect::Value::F32(v) => vrl::value::Value::Float( + NotNan::new(f64::from(*v)).map_err(|_e| "Float number cannot be Nan")?, + ), + prost_reflect::Value::F64(v) => { + vrl::value::Value::Float(NotNan::new(*v).map_err(|_e| "F64 number cannot be Nan")?) + } + prost_reflect::Value::String(v) => vrl::value::Value::from(v.as_str()), + prost_reflect::Value::Bytes(v) => vrl::value::Value::from(v.clone()), + prost_reflect::Value::EnumNumber(v) => { + if let Some(field_descriptor) = field_descriptor { + let kind = field_descriptor.kind(); + let enum_desc = kind.as_enum().ok_or_else(|| { + format!( + "Internal error while parsing protobuf enum. Field descriptor: {:?}", + field_descriptor + ) + })?; + vrl::value::Value::from( + enum_desc + .get_value(*v) + .ok_or_else(|| { + format!("The number {} cannot be in '{}'", v, enum_desc.name()) + })? + .name(), + ) + } else { + Err("Expected valid field descriptor")? + } + } + prost_reflect::Value::Message(v) => { + let mut obj_map = BTreeMap::new(); + for field_desc in v.descriptor().fields() { + let field_value = v.get_field(&field_desc); + let out = to_vrl(field_value.as_ref(), Some(&field_desc))?; + obj_map.insert(field_desc.name().to_string(), out); + } + vrl::value::Value::from(obj_map) + } + prost_reflect::Value::List(v) => { + let vec = v + .iter() + .map(|o| to_vrl(o, field_descriptor)) + .collect::, vector_common::Error>>()?; + vrl::value::Value::from(vec) + } + prost_reflect::Value::Map(v) => { + if let Some(field_descriptor) = field_descriptor { + let kind = field_descriptor.kind(); + let message_desc = kind.as_message().ok_or_else(|| { + format!( + "Internal error while parsing protobuf field descriptor: {:?}", + field_descriptor + ) + })?; + vrl::value::Value::from( + v.iter() + .map(|kv| { + Ok(( + kv.0.as_str() + .ok_or_else(|| { + format!( + "Internal error while parsing protobuf map. Field descriptor: {:?}", + field_descriptor + ) + })? + .to_string(), + to_vrl(kv.1, Some(&message_desc.map_entry_value_field()))?, + )) + }) + .collect::>>()?, + ) + } else { + Err("Expected valid field descriptor")? + } + } + }; + Ok(vrl_value) +} + +#[cfg(test)] +mod tests { + // TODO: add test for bad file path & invalid message_type + + use std::path::PathBuf; + use std::{env, fs}; + use vector_core::config::log_schema; + + use super::*; + + fn test_data_dir() -> PathBuf { + PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .join("tests/data/decoding/protobuf") + } + + fn parse_and_validate( + protobuf_bin_message: String, + protobuf_desc_path: PathBuf, + message_type: &str, + validate_log: fn(&LogEvent), + ) { + let input = Bytes::from(protobuf_bin_message); + let message_descriptor = ProtobufDeserializer::get_message_descriptor( + &protobuf_desc_path, + message_type.to_string(), + ) + .unwrap(); + let deserializer = ProtobufDeserializer::new(message_descriptor); + + for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { + let events = deserializer.parse(input.clone(), namespace).unwrap(); + let mut events = events.into_iter(); + + { + let event = events.next().unwrap(); + let log = event.as_log(); + validate_log(log); + assert_eq!( + log.get(log_schema().timestamp_key_target_path().unwrap()) + .is_some(), + namespace == LogNamespace::Legacy + ); + } + + assert_eq!(events.next(), None); + } + } + + #[test] + fn deserialize_protobuf() { + let protobuf_bin_message_path = test_data_dir().join("person_someone.pb"); + let protobuf_desc_path = test_data_dir().join("test_protobuf.desc"); + let message_type = "test_protobuf.Person"; + let validate_log = |log: &LogEvent| { + assert_eq!(log["name"], "someone".into()); + assert_eq!( + log["phones"].as_array().unwrap()[0].as_object().unwrap()["number"] + .as_str() + .unwrap(), + "123456" + ); + }; + + parse_and_validate( + fs::read_to_string(protobuf_bin_message_path).unwrap(), + protobuf_desc_path, + message_type, + validate_log, + ); + } + + #[test] + fn deserialize_protobuf3() { + let protobuf_bin_message_path = test_data_dir().join("person_someone3.pb"); + let protobuf_desc_path = test_data_dir().join("test_protobuf3.desc"); + let message_type = "test_protobuf3.Person"; + let validate_log = |log: &LogEvent| { + assert_eq!(log["name"], "someone".into()); + assert_eq!( + log["phones"].as_array().unwrap()[0].as_object().unwrap()["number"] + .as_str() + .unwrap(), + "1234" + ); + assert_eq!( + log["data"].as_object().unwrap()["data_phone"], + "HOME".into() + ); + }; + + parse_and_validate( + fs::read_to_string(protobuf_bin_message_path).unwrap(), + protobuf_desc_path, + message_type, + validate_log, + ); + } + + #[test] + fn deserialize_empty_buffer() { + let protobuf_bin_message = "".to_string(); + let protobuf_desc_path = test_data_dir().join("test_protobuf.desc"); + let message_type = "test_protobuf.Person"; + let validate_log = |log: &LogEvent| { + assert_eq!(log["name"], "".into()); + }; + + parse_and_validate( + protobuf_bin_message, + protobuf_desc_path, + message_type, + validate_log, + ); + } + + #[test] + fn deserialize_error_invalid_protobuf() { + let input = Bytes::from("{ foo"); + let message_descriptor = ProtobufDeserializer::get_message_descriptor( + &test_data_dir().join("test_protobuf.desc"), + "test_protobuf.Person".to_string(), + ) + .unwrap(); + let deserializer = ProtobufDeserializer::new(message_descriptor); + + for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { + assert!(deserializer.parse(input.clone(), namespace).is_err()); + } + } +} diff --git a/lib/codecs/src/decoding/mod.rs b/lib/codecs/src/decoding/mod.rs index 96b3d04dee82c..0539a6a67d2cd 100644 --- a/lib/codecs/src/decoding/mod.rs +++ b/lib/codecs/src/decoding/mod.rs @@ -11,7 +11,8 @@ pub use format::{ BoxedDeserializer, BytesDeserializer, BytesDeserializerConfig, GelfDeserializer, GelfDeserializerConfig, GelfDeserializerOptions, JsonDeserializer, JsonDeserializerConfig, JsonDeserializerOptions, NativeDeserializer, NativeDeserializerConfig, NativeJsonDeserializer, - NativeJsonDeserializerConfig, NativeJsonDeserializerOptions, + NativeJsonDeserializerConfig, NativeJsonDeserializerOptions, ProtobufDeserializer, + ProtobufDeserializerConfig, }; #[cfg(feature = "syslog")] pub use format::{SyslogDeserializer, SyslogDeserializerConfig, SyslogDeserializerOptions}; @@ -200,6 +201,11 @@ pub enum DeserializerConfig { /// [json]: https://www.json.org/ Json(JsonDeserializerConfig), + /// Decodes the raw bytes as [protobuf][protobuf]. + /// + /// [protobuf]: https://protobuf.dev/ + Protobuf(ProtobufDeserializerConfig), + #[cfg(feature = "syslog")] /// Decodes the raw bytes as a Syslog message. /// @@ -275,6 +281,7 @@ impl DeserializerConfig { match self { DeserializerConfig::Bytes => Deserializer::Bytes(BytesDeserializerConfig.build()), DeserializerConfig::Json(config) => Deserializer::Json(config.build()), + DeserializerConfig::Protobuf(config) => Deserializer::Protobuf(config.build()), #[cfg(feature = "syslog")] DeserializerConfig::Syslog(config) => Deserializer::Syslog(config.build()), DeserializerConfig::Native => Deserializer::Native(NativeDeserializerConfig.build()), @@ -293,6 +300,7 @@ impl DeserializerConfig { | DeserializerConfig::NativeJson(_) => { FramingConfig::NewlineDelimited(Default::default()) } + DeserializerConfig::Protobuf(_) => FramingConfig::Bytes, #[cfg(feature = "syslog")] DeserializerConfig::Syslog(_) => FramingConfig::NewlineDelimited(Default::default()), } @@ -303,6 +311,7 @@ impl DeserializerConfig { match self { DeserializerConfig::Bytes => BytesDeserializerConfig.output_type(), DeserializerConfig::Json(config) => config.output_type(), + DeserializerConfig::Protobuf(config) => config.output_type(), #[cfg(feature = "syslog")] DeserializerConfig::Syslog(config) => config.output_type(), DeserializerConfig::Native => NativeDeserializerConfig.output_type(), @@ -316,6 +325,7 @@ impl DeserializerConfig { match self { DeserializerConfig::Bytes => BytesDeserializerConfig.schema_definition(log_namespace), DeserializerConfig::Json(config) => config.schema_definition(log_namespace), + DeserializerConfig::Protobuf(config) => config.schema_definition(log_namespace), #[cfg(feature = "syslog")] DeserializerConfig::Syslog(config) => config.schema_definition(log_namespace), DeserializerConfig::Native => NativeDeserializerConfig.schema_definition(log_namespace), @@ -344,6 +354,7 @@ impl DeserializerConfig { }), ) => "application/json", (DeserializerConfig::Native, _) => "application/octet-stream", + (DeserializerConfig::Protobuf(_), _) => "application/octet-stream", ( DeserializerConfig::Json(_) | DeserializerConfig::NativeJson(_) @@ -364,6 +375,8 @@ pub enum Deserializer { Bytes(BytesDeserializer), /// Uses a `JsonDeserializer` for deserialization. Json(JsonDeserializer), + /// Uses a `ProtobufDeserializer` for deserialization. + Protobuf(ProtobufDeserializer), #[cfg(feature = "syslog")] /// Uses a `SyslogDeserializer` for deserialization. Syslog(SyslogDeserializer), @@ -386,6 +399,7 @@ impl format::Deserializer for Deserializer { match self { Deserializer::Bytes(deserializer) => deserializer.parse(bytes, log_namespace), Deserializer::Json(deserializer) => deserializer.parse(bytes, log_namespace), + Deserializer::Protobuf(deserializer) => deserializer.parse(bytes, log_namespace), #[cfg(feature = "syslog")] Deserializer::Syslog(deserializer) => deserializer.parse(bytes, log_namespace), Deserializer::Native(deserializer) => deserializer.parse(bytes, log_namespace), diff --git a/lib/codecs/tests/data/decoding/protobuf/person_someone.pb b/lib/codecs/tests/data/decoding/protobuf/person_someone.pb new file mode 100644 index 0000000000000..46b78f47969af --- /dev/null +++ b/lib/codecs/tests/data/decoding/protobuf/person_someone.pb @@ -0,0 +1,3 @@ + +someone" +123456 \ No newline at end of file diff --git a/lib/codecs/tests/data/decoding/protobuf/person_someone.txt b/lib/codecs/tests/data/decoding/protobuf/person_someone.txt new file mode 100644 index 0000000000000..72e26e70a16df --- /dev/null +++ b/lib/codecs/tests/data/decoding/protobuf/person_someone.txt @@ -0,0 +1,2 @@ +debug print of person_someone.pb with prost +Person { name: Some("someone"), id: None, email: None, phones: [PhoneNumber { number: Some("123456"), r#type: None }] } diff --git a/lib/codecs/tests/data/decoding/protobuf/person_someone3.pb b/lib/codecs/tests/data/decoding/protobuf/person_someone3.pb new file mode 100644 index 0000000000000000000000000000000000000000..80e4ef6f4dfb6694d4203b9212e567241810a41c GIT binary patch literal 35 ocmd;LFV4?R&Cg3!;^X2FUSD$1Q@kAxL6F0j70I&22iU0rr literal 0 HcmV?d00001 diff --git a/lib/codecs/tests/data/decoding/protobuf/person_someone3.txt b/lib/codecs/tests/data/decoding/protobuf/person_someone3.txt new file mode 100644 index 0000000000000..0a5086c628414 --- /dev/null +++ b/lib/codecs/tests/data/decoding/protobuf/person_someone3.txt @@ -0,0 +1,2 @@ +debug print of person_someone3.pb with prost +Person { name: Some("someone"), id: None, email: None, data: {"data_phone": Home}, phones: [PhoneNumber { number: Some("1234"), r#type: Some(Mobile) }] } diff --git a/lib/codecs/tests/data/decoding/protobuf/test_protobuf.desc b/lib/codecs/tests/data/decoding/protobuf/test_protobuf.desc new file mode 100644 index 0000000000000000000000000000000000000000..43e7acf6cf7716af7c2a751c2ece9283a5033f47 GIT binary patch literal 1183 zcmaJ<+int36rFQn2KF#890r&|FO;Vyji`Oo_+pbL-fA0S;-j`f^dJ9EkiFDzt}ZWTHf3^34O@W; zL*!mwoxlBX5jv($Ru&45K3~n=hjQ|Y{MYmOI@xT_)AWaVg29q>b791-(-2%bH?GJJjwCl^kPady;(7g9hjb382RHnqshAdE;;h0y{9tx^u!*Fi}O zi|ZAJ-1nQM9I&|~=Qf+Vl1t_xw~Z<^+U9b0or?dYO})%Y*+xTslvQGgZG^g6VeBIe zx)nxx7!50oorDL6r5)&weqPf2L!BKv;Q|N!if8>`RAJb44y$%V9eDh#;yvKgr};lZ z9LY&Gm{cegmH((vDDsONQYaG*4hkjbjgRfQ3Zn;n&U~)C^+COB%?1BpEVC2=Y5{{rq%g!BLa literal 0 HcmV?d00001 diff --git a/lib/codecs/tests/data/decoding/protobuf/test_protobuf.proto b/lib/codecs/tests/data/decoding/protobuf/test_protobuf.proto new file mode 100644 index 0000000000000..a4cb111f18ec5 --- /dev/null +++ b/lib/codecs/tests/data/decoding/protobuf/test_protobuf.proto @@ -0,0 +1,26 @@ +syntax = "proto2"; + +package test_protobuf; + +message Person { + optional string name = 1; + optional int32 id = 2; + optional string email = 3; + + enum PhoneType { + MOBILE = 0; + HOME = 1; + WORK = 2; + } + + message PhoneNumber { + optional string number = 1; + optional PhoneType type = 2 [default = HOME]; + } + + repeated PhoneNumber phones = 4; +} + +message AddressBook { + repeated Person people = 1; +} diff --git a/lib/codecs/tests/data/decoding/protobuf/test_protobuf3.desc b/lib/codecs/tests/data/decoding/protobuf/test_protobuf3.desc new file mode 100644 index 0000000000000000000000000000000000000000..9fcfec71b44b8c52c45dbbc3a2b179e8c32d0529 GIT binary patch literal 1429 zcma)6+iKfD5Y=9^vPZti>zicf;`%92@IzBd8|Xt#1L-AA5QWkw<6xDB*p{)Kg8!h; z{h|I!KczFPn*$B>ZO@)FXJ&Rrf?qDQw$WxgS*_!3d~^TpbYxDt_V9N2M}WGH)|+^# zyC9d-MdY&qJ1JjSp+P?x(@o&FGoRzNeZfLJ(`5j5w3yz`ePQH0OdwIuLC&VzsV@iO z#C+pk=;%MUJCLB$iyb_Kk4t@+t%^Xdf z9(922P@`pD6$xirEju1mK@msIf~-RgoCKl~NGLFt0M%M9N7|LuoW|8eTq-ciUAL0U z5mxr(RAyyQ&WT64DO6ygbxLQN5HyoIv=b{~3iaf1l8GR`h9@fp#`o~5%>rY%*Xk7* z-^4pS%I%PD$U#oigc^HxOd+mz3ZC@ceu2@hade!{2|c&{jTrfq|{yFylx$ zc#?);&UPLq4aycxqDkA$R!gEuyPKIvG-;y=Z86i*r!A)~reXM{rZHzU*nlS?2+%x8 ccLx!g2kGu0!mvOmtBMHCgZ{ydGQZx_zvT?XAOHXW literal 0 HcmV?d00001 diff --git a/lib/codecs/tests/data/decoding/protobuf/test_protobuf3.proto b/lib/codecs/tests/data/decoding/protobuf/test_protobuf3.proto new file mode 100644 index 0000000000000..672a05d2fac9a --- /dev/null +++ b/lib/codecs/tests/data/decoding/protobuf/test_protobuf3.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package test_protobuf3; + +message Person { + optional string name = 1; + optional int32 id = 2; + optional string email = 3; + + enum PhoneType { + MOBILE = 0; + HOME = 1; + WORK = 2; + } + + message PhoneNumber { + optional string number = 1; + optional PhoneType type = 2; + } + + map data = 4; + repeated PhoneNumber phones = 5; +} + +message AddressBook { + repeated Person people = 1; +} diff --git a/src/components/validation/resources/mod.rs b/src/components/validation/resources/mod.rs index a22d6fc324dbd..0ed2060b60be0 100644 --- a/src/components/validation/resources/mod.rs +++ b/src/components/validation/resources/mod.rs @@ -142,6 +142,7 @@ fn deserializer_config_to_serializer(config: &DeserializerConfig) -> encoding::S // `message` field... but it's close enough for now. DeserializerConfig::Bytes => SerializerConfig::Text(TextSerializerConfig::default()), DeserializerConfig::Json { .. } => SerializerConfig::Json(JsonSerializerConfig::default()), + DeserializerConfig::Protobuf(_) => unimplemented!(), // TODO: We need to create an Avro serializer because, certainly, for any source decoding // the data as Avro, we can't possibly send anything else without the source just // immediately barfing. diff --git a/website/cue/reference/components/sources/base/amqp.cue b/website/cue/reference/components/sources/base/amqp.cue index 93d2becda8f58..5dca16763beee 100644 --- a/website/cue/reference/components/sources/base/amqp.cue +++ b/website/cue/reference/components/sources/base/amqp.cue @@ -83,6 +83,11 @@ base: components: sources: amqp: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -95,6 +100,12 @@ base: components: sources: amqp: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -127,6 +138,12 @@ base: components: sources: amqp: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue index 8b17bfc8a5fbf..abc1d6106fc4f 100644 --- a/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue @@ -86,6 +86,11 @@ base: components: sources: aws_kinesis_firehose: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -98,6 +103,12 @@ base: components: sources: aws_kinesis_firehose: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -130,6 +141,12 @@ base: components: sources: aws_kinesis_firehose: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/aws_s3.cue b/website/cue/reference/components/sources/base/aws_s3.cue index 62fa9888d87b3..c382a52257f67 100644 --- a/website/cue/reference/components/sources/base/aws_s3.cue +++ b/website/cue/reference/components/sources/base/aws_s3.cue @@ -181,6 +181,11 @@ base: components: sources: aws_s3: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -193,6 +198,12 @@ base: components: sources: aws_s3: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -225,6 +236,12 @@ base: components: sources: aws_s3: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/aws_sqs.cue b/website/cue/reference/components/sources/base/aws_sqs.cue index 35cdf11ac23a4..c9f69ce7b493d 100644 --- a/website/cue/reference/components/sources/base/aws_sqs.cue +++ b/website/cue/reference/components/sources/base/aws_sqs.cue @@ -176,6 +176,11 @@ base: components: sources: aws_sqs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -188,6 +193,12 @@ base: components: sources: aws_sqs: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -220,6 +231,12 @@ base: components: sources: aws_sqs: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/datadog_agent.cue b/website/cue/reference/components/sources/base/datadog_agent.cue index 4f625a5717ce5..c4650c18517fb 100644 --- a/website/cue/reference/components/sources/base/datadog_agent.cue +++ b/website/cue/reference/components/sources/base/datadog_agent.cue @@ -68,6 +68,11 @@ base: components: sources: datadog_agent: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -80,6 +85,12 @@ base: components: sources: datadog_agent: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -112,6 +123,12 @@ base: components: sources: datadog_agent: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/demo_logs.cue b/website/cue/reference/components/sources/base/demo_logs.cue index d8366fa1c5ec9..80d67e4a3071c 100644 --- a/website/cue/reference/components/sources/base/demo_logs.cue +++ b/website/cue/reference/components/sources/base/demo_logs.cue @@ -47,6 +47,11 @@ base: components: sources: demo_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -59,6 +64,12 @@ base: components: sources: demo_logs: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -91,6 +102,12 @@ base: components: sources: demo_logs: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/exec.cue b/website/cue/reference/components/sources/base/exec.cue index cc0e2ab6618e6..fe9857a6626ae 100644 --- a/website/cue/reference/components/sources/base/exec.cue +++ b/website/cue/reference/components/sources/base/exec.cue @@ -43,6 +43,11 @@ base: components: sources: exec: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -55,6 +60,12 @@ base: components: sources: exec: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -87,6 +98,12 @@ base: components: sources: exec: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/file_descriptor.cue b/website/cue/reference/components/sources/base/file_descriptor.cue index 6eaed568f2526..4c65fd3d0dbed 100644 --- a/website/cue/reference/components/sources/base/file_descriptor.cue +++ b/website/cue/reference/components/sources/base/file_descriptor.cue @@ -38,6 +38,11 @@ base: components: sources: file_descriptor: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -50,6 +55,12 @@ base: components: sources: file_descriptor: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -82,6 +93,12 @@ base: components: sources: file_descriptor: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/gcp_pubsub.cue b/website/cue/reference/components/sources/base/gcp_pubsub.cue index 767c7587aa0a4..73bc51dbb2f83 100644 --- a/website/cue/reference/components/sources/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sources/base/gcp_pubsub.cue @@ -114,6 +114,11 @@ base: components: sources: gcp_pubsub: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -126,6 +131,12 @@ base: components: sources: gcp_pubsub: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -158,6 +169,12 @@ base: components: sources: gcp_pubsub: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/heroku_logs.cue b/website/cue/reference/components/sources/base/heroku_logs.cue index 982d0343ac1d1..e5eab4e11a1e6 100644 --- a/website/cue/reference/components/sources/base/heroku_logs.cue +++ b/website/cue/reference/components/sources/base/heroku_logs.cue @@ -80,6 +80,11 @@ base: components: sources: heroku_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -92,6 +97,12 @@ base: components: sources: heroku_logs: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -124,6 +135,12 @@ base: components: sources: heroku_logs: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/http.cue b/website/cue/reference/components/sources/base/http.cue index b5aac87c8c2ff..e031e1df64760 100644 --- a/website/cue/reference/components/sources/base/http.cue +++ b/website/cue/reference/components/sources/base/http.cue @@ -82,6 +82,11 @@ base: components: sources: http: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -93,6 +98,12 @@ base: components: sources: http: configuration: { """ } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -125,6 +136,12 @@ base: components: sources: http: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/http_client.cue b/website/cue/reference/components/sources/base/http_client.cue index a7e0fa5f37d23..d84f7f43839c7 100644 --- a/website/cue/reference/components/sources/base/http_client.cue +++ b/website/cue/reference/components/sources/base/http_client.cue @@ -80,6 +80,11 @@ base: components: sources: http_client: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -92,6 +97,12 @@ base: components: sources: http_client: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -124,6 +135,12 @@ base: components: sources: http_client: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/http_server.cue b/website/cue/reference/components/sources/base/http_server.cue index 6cda10ae713b2..5341bea4c3d3f 100644 --- a/website/cue/reference/components/sources/base/http_server.cue +++ b/website/cue/reference/components/sources/base/http_server.cue @@ -82,6 +82,11 @@ base: components: sources: http_server: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -93,6 +98,12 @@ base: components: sources: http_server: configuration: { """ } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -125,6 +136,12 @@ base: components: sources: http_server: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/kafka.cue b/website/cue/reference/components/sources/base/kafka.cue index 9ee5b6e6b2dc6..1791dc5500856 100644 --- a/website/cue/reference/components/sources/base/kafka.cue +++ b/website/cue/reference/components/sources/base/kafka.cue @@ -92,6 +92,11 @@ base: components: sources: kafka: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -104,6 +109,12 @@ base: components: sources: kafka: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -136,6 +147,12 @@ base: components: sources: kafka: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/nats.cue b/website/cue/reference/components/sources/base/nats.cue index 6cb2b3e956dc9..3620382321118 100644 --- a/website/cue/reference/components/sources/base/nats.cue +++ b/website/cue/reference/components/sources/base/nats.cue @@ -135,6 +135,11 @@ base: components: sources: nats: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -147,6 +152,12 @@ base: components: sources: nats: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -179,6 +190,12 @@ base: components: sources: nats: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/redis.cue b/website/cue/reference/components/sources/base/redis.cue index 1e81399a5f3c4..11e4cb69b2d40 100644 --- a/website/cue/reference/components/sources/base/redis.cue +++ b/website/cue/reference/components/sources/base/redis.cue @@ -53,6 +53,11 @@ base: components: sources: redis: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -65,6 +70,12 @@ base: components: sources: redis: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -97,6 +108,12 @@ base: components: sources: redis: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/socket.cue b/website/cue/reference/components/sources/base/socket.cue index d49d70ffb589f..044035db0b789 100644 --- a/website/cue/reference/components/sources/base/socket.cue +++ b/website/cue/reference/components/sources/base/socket.cue @@ -55,6 +55,11 @@ base: components: sources: socket: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -67,6 +72,12 @@ base: components: sources: socket: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -99,6 +110,12 @@ base: components: sources: socket: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\"" diff --git a/website/cue/reference/components/sources/base/stdin.cue b/website/cue/reference/components/sources/base/stdin.cue index 72141036aa562..bf388e17c605d 100644 --- a/website/cue/reference/components/sources/base/stdin.cue +++ b/website/cue/reference/components/sources/base/stdin.cue @@ -38,6 +38,11 @@ base: components: sources: stdin: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Decodes the raw bytes as [protobuf][protobuf]. + + [protobuf]: https://protobuf.dev/ + """ syslog: """ Decodes the raw bytes as a Syslog message. @@ -50,6 +55,12 @@ base: components: sources: stdin: configuration: { } } } + desc_file: { + description: "Path to desc file" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } gelf: { description: "GELF-specific decoding options." relevant_when: "codec = \"gelf\"" @@ -82,6 +93,12 @@ base: components: sources: stdin: configuration: { type: bool: default: true } } + message_type: { + description: "message type. e.g package.message" + relevant_when: "codec = \"protobuf\"" + required: true + type: string: {} + } native_json: { description: "Vector's native JSON-specific decoding options." relevant_when: "codec = \"native_json\""