From fd6ba5144014781bc867d21c293577df0a2d0010 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 21 Sep 2023 12:36:13 -0400 Subject: [PATCH] chore(docs): Covnert a few more configs to YAML --- .../content/en/blog/vector-remap-language.md | 49 +++++---- .../en/guides/level-up/transformation.md | 101 ++++++++++-------- website/cue/reference/components/sinks.cue | 19 ++-- 3 files changed, 93 insertions(+), 76 deletions(-) diff --git a/website/content/en/blog/vector-remap-language.md b/website/content/en/blog/vector-remap-language.md index febec744de6dc..d580e612ac4bb 100644 --- a/website/content/en/blog/vector-remap-language.md +++ b/website/content/en/blog/vector-remap-language.md @@ -110,29 +110,34 @@ Parsing this log, without VRL, looks like this: {{< tabs default="Vector" >}} {{< tab title="Vector" >}} -```toml title="vector.toml" +```yaml title="vector.yaml" # ... sources ... -# Parse the internal Syslog log -[transforms.parse_syslog] -type = "regex_parser" -inputs = ["parse_docker"] -patterns = ['^(?P\S+) (?P\S+) (?P\S+) \[(?[\w:/]+\s[+\-]\d{4})\] "(?\S+) (?.+?) (?\S+)" (?\d{3}) (?\S+)$'] -field = "log" - -# Remove Docker time and log fields since they are duplicative -[transform.remove_log] -type = "remove_fields" -inputs = ["parse_syslog"] -fields = ["time", "log"] - -# Coerce parsed fields -[transforms.coerce_fields] -type = "coercer" -inputs = ["remove_log"] -types.timestamp = "timestamp" -types.status = "int" -types.bytes_out = "int" +transforms: + parse_syslog: + type: regex_parser + inputs: + - parse_docker + patterns: + - '^(?P\S+) (?P\S+) (?P\S+) \[(?[\w:/]+\s[+\-]\d{4})\] "(?\S+) (?.+?) (?\S+)" (?\d{3}) (?\S+)$' + field: log + + remove_log: + type: remove_fields + inputs: + - parse_syslog + fields: + - time + - log + + coerce_fields: + type: coercer + inputs: + - remove_log + types: + timestamp: timestamp + status: int + bytes_out: int # ... sinks ... ``` @@ -224,7 +229,7 @@ transformation. Instead of conflating these concerns, like Logstash, Fluentd, and others, Vector separates them, allowing you to choose your preferred configuration language -(TOML, YAML, or JSON) while offering a purpose-built language for data +(YAML, TOML or JSON) while offering a purpose-built language for data transformation (VRL). But is VRL really necessary? Couldn't you leverage Lua, JavaScript, or any other existing language? diff --git a/website/content/en/guides/level-up/transformation.md b/website/content/en/guides/level-up/transformation.md index 643c10170868a..98db0049d9f73 100644 --- a/website/content/en/guides/level-up/transformation.md +++ b/website/content/en/guides/level-up/transformation.md @@ -56,37 +56,44 @@ create a simple topology consisting of three components: This configuration defines that topology: -```toml title="vector.toml" -[sources.logs] - type = "demo_logs" - format = "syslog" - interval = 0.1 - -[transforms.modify] - type = "remap" - inputs = ["logs"] - source = ''' - # Parse Syslog input. The "!" means that the script should abort on error. - . = parse_syslog!(.message) - ''' - -[sinks.out] - type = "console" - inputs = ["modify"] - encoding.codec = "json" +```yaml title="vector.yaml" +sources: + logs: + type: demo_logs + format: syslog + interval: 0.1 + +transforms: + modify: + type: remap + inputs: + - logs + source: | + # Parse Syslog input. The "!" means that the script should abort on error. + . = parse_syslog!(.message) + +sinks: + out: + type: console + inputs: + - modify + encoding: + codec: json ``` {{< info >}} -Although we're using [TOML][urls.toml] for the configuration here, Vector also -supports JSON and YAML. +Although we're using [YAML][urls.yaml] for the configuration here, Vector also +supports [TOML][urls.toml] and [JSON][urls.json]. [urls.toml]: https://github.com/toml-lang/toml +[urls.yaml]: https://yaml.org +[urls.json]: https://www.json.org/json-en.html {{< /info >}} To start Vector using this topology: ```bash -vector --config-toml /etc/vector/vector.toml +vector --config /etc/vector/vector.yaml ``` You should see lines like this emitted via stdout (formatted for readability @@ -109,31 +116,33 @@ So far, we've gotten Vector to *parse* the Syslog data but we're not yet *modifying* that data. So let's update the `source` script of our `remap` transform to make some ad hoc transformations: -```toml -[transforms.modify] - type = "remap" - inputs = ["logs"] - source = ''' - . = parse_syslog!(.message) - - # Convert the timestamp to a Unix timestamp, aborting on error - .timestamp = to_unix_timestamp!(.timestamp) - - # Remove the "facility" and "procid" fields - del(.facility); del(.procid) - - # Replace the "msgid" field with a unique ID - .msgid = uuid_v4() - - # If the log message contains the phrase "Great Scott!", set the new field - # "critical" to true, otherwise set it to false. If the "contains" function - # errors, log the error (instead of aborting the script, as above). - if (is_critical, err = contains(.message, "Great Scott!"); err != null) { - log(err, level: "error") - } - - .critical = is_critical - ''' +```yaml +transforms: + modify: + type: remap + inputs: + - logs + source: | + . = parse_syslog!(.message) + + # Convert the timestamp to a Unix timestamp, aborting on error + .timestamp = to_unix_timestamp!(.timestamp) + + # Remove the "facility" and "procid" fields + del(.facility) + del(.procid) + + # Replace the "msgid" field with a unique ID + .msgid = uuid_v4() + + # If the log message contains the phrase "Great Scott!", set the new field + # "critical" to true, otherwise set it to false. If the "contains" function + # errors, log the error (instead of aborting the script, as above). + if (is_critical, err = contains(.message, "Great Scott!"); err != null) { + log(err, level: "error") + } + + .critical = is_critical ``` A few things to notice about this script: diff --git a/website/cue/reference/components/sinks.cue b/website/cue/reference/components/sinks.cue index 32bcd0821e670..90df0f84ab518 100644 --- a/website/cue/reference/components/sinks.cue +++ b/website/cue/reference/components/sinks.cue @@ -601,10 +601,11 @@ components: sinks: [Name=string]: { If Adaptive Request Concurrency is not for you, you can manually set static concurrency limits by specifying an integer for `request.concurrency`: - ```toml title="vector.toml" - [sinks.my-sink] - request.concurrency = 10 - ``` + ```yaml title="vector.yaml" + sinks: + my-sink: + request: + concurrency: 10 """ }, { @@ -614,10 +615,12 @@ components: sinks: [Name=string]: { throughput via the `request.rate_limit_duration_secs` and `request.rate_limit_num` options. - ```toml title="vector.toml" - [sinks.my-sink] - request.rate_limit_duration_secs = 1 - request.rate_limit_num = 10 + ```yaml title="vector.yaml" + sinks: + my-sink: + request: + rate_limit_duration_secs: 1 + rate_limit_num: 10 ``` These will apply to both `adaptive` and fixed `request.concurrency` values.