From 3b931e6a5408e2f03a6620591cc63794659daeb3 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 8 Nov 2023 13:37:07 -0500 Subject: [PATCH] fix test config fix yaml test config and also format the code fix wrong usage log.get, now all tests are passing Ran cargo vdev fmt Fix reference in to_metrics_metadata in log_to_metric.rs Convert type errors to single error with type. Added test for multiple metrics but it is failing. Additional attempts to get the multiple_metadata_metrics working. fix(playground): fix playground vrl version and link (#19119) * fix(playground): fix playground vrl version and link * Ran cargo vdev fmt * remove redundant line * replace clone() with as_ref() whenever possible * Ran cargo vdev fmt chore(website): Fix commenting step on workflow (#19134) * fix: extra env vars into comment step * fix: add sleep to allow branch to connect * fix: update where branch name and sanitized branch name are used Remove multiple metrics test. chore(ci): Bump bufbuild/buf-setup-action from 1.27.2 to 1.28.0 (#19137) Bumps [bufbuild/buf-setup-action](https://github.com/bufbuild/buf-setup-action) from 1.27.2 to 1.28.0. - [Release notes](https://github.com/bufbuild/buf-setup-action/releases) - [Commits](https://github.com/bufbuild/buf-setup-action/compare/v1.27.2...v1.28.0) --- updated-dependencies: - dependency-name: bufbuild/buf-setup-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> chore(deps): Bump the clap group with 1 update (#19127) Bumps the clap group with 1 update: [clap](https://github.com/clap-rs/clap). - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.4.7...v4.4.8) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-patch dependency-group: clap ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> chore(releasing): Add known issue for Datadog Metrics sink in v0.34.0 (#19122) * chore(releasing): Add known issue for Datadog Metrics sink in v0.34.0 Ref: https://github.com/vectordotdev/vector/issues/19110 Signed-off-by: Jesse Szwedko * Update website/cue/reference/releases/0.34.0.cue Co-authored-by: Brett Blue <84536271+brett0000FF@users.noreply.github.com> --------- Signed-off-by: Jesse Szwedko Co-authored-by: Brett Blue <84536271+brett0000FF@users.noreply.github.com> chore(deps): Bump proptest from 1.3.1 to 1.4.0 (#19131) Bumps [proptest](https://github.com/proptest-rs/proptest) from 1.3.1 to 1.4.0. - [Release notes](https://github.com/proptest-rs/proptest/releases) - [Changelog](https://github.com/proptest-rs/proptest/blob/master/CHANGELOG.md) - [Commits](https://github.com/proptest-rs/proptest/compare/v1.3.1...v1.4.0) --- updated-dependencies: - dependency-name: proptest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> chore(deps): Bump env_logger from 0.10.0 to 0.10.1 (#19130) Bumps [env_logger](https://github.com/rust-cli/env_logger) from 0.10.0 to 0.10.1. - [Release notes](https://github.com/rust-cli/env_logger/releases) - [Changelog](https://github.com/rust-cli/env_logger/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-cli/env_logger/compare/v0.10.0...v0.10.1) --- updated-dependencies: - dependency-name: env_logger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> chore(docs): Add alpha to traces and beta to metrics in descriptions (#19139) add alpha to traces and beta to metrics in descriptions Update README.md (#19142) chore(deps): Bump hdrhistogram from 7.5.2 to 7.5.3 (#19129) Bumps [hdrhistogram](https://github.com/HdrHistogram/HdrHistogram_rust) from 7.5.2 to 7.5.3. - [Release notes](https://github.com/HdrHistogram/HdrHistogram_rust/releases) - [Changelog](https://github.com/HdrHistogram/HdrHistogram_rust/blob/main/CHANGELOG.md) - [Commits](https://github.com/HdrHistogram/HdrHistogram_rust/compare/v7.5.2...v7.5.3) --- updated-dependencies: - dependency-name: hdrhistogram dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> enhancement(file sink, aws_s3 sink, gcp_cloud_storage): configurable filename timezone (#18506) * add TzOffset as File Sink configuration * integrate TzOffset into File Sink * apply tz offset to all log event in render_timestamp. * added TzOffset tests * adding chrono-tz for parsing timezones * rename tz_offset to path_tz. timezones are safer than offsets * update tz_offset references to path_tz * cargo fmt * remove unnecessary commented out code. fmt and generate-component-docs * clippy suggestions and remove TryFrom<&str> - serde handles converting to String * rename Template config option `path_tz` to `timezone` * move `path_tz.rs` to `src/config` preparing for applying the same to `aws_s3` sink for filename timezone * update doc configuration description for path_tz * fix wrong method name * AWS and GCS filename timezone support * remove custom tz config * use VRL's timezone config * pass around SinkContext * use TzOffset to pass down to request builders. VRL's TimeZone can't be hash derived * make key_prefix timezone aware and use Option `or` syntax * move tz to offset conversion codes to sink util * remove empty line * update timezone docs in vector-config * get timezone and convert to offset in one go in FileSink * just pass the sinkconfig directly * vector_common to vector_lib * configurable_component is in vector_lib now * lookup to vector_lib * fix aws s3 integration test. pass the context to build_processor in tests * formatting * add sinkcontext to FileSink in file tests * key_prefix is expected to be a template. no need for into Update documentation Updated documentation to include example enhancement(networking, sinks): add full jitter to retry backoff policy (#19106) * enhancement(networking, sinks): add full jitter to retry backoff policy * fmt * fix tests * add test * fix * force ci fix(file source, kubernetes_logs source, file sink): make file internal metric tag opt-in (#19145) * fix(file source, kubernetes_logs source, file sink): make file internal metric tag opt-in * update cue * fix tests fix(datadog_metrics sink): evaluate series v1 env var at runtime (#19148) fix(datadog_metrics sink): evaluate v1 env var at runtime chore(website): WEB-4247 | Update references from s3 to setup.vector.dev (#19149) feat: update references from s3 to setup.vector.dev fix(ARC, networking): improve request settings (#19101) * fix(ARC, networking): improve request settings * fix spelling * change defaults * refactor * self-review * clippy * update default overrides * fmt nit * add upgrade guide entry Update src/transforms/log_to_metric.rs Co-authored-by: Pavlos Rontidis Tweak and generate docs Tweak and generate docs Update website/cue/reference/components/transforms/base/log_to_metric.cue Co-authored-by: Heston Hoffman Update website/cue/reference/components/transforms/base/log_to_metric.cue Co-authored-by: Heston Hoffman Update website/cue/reference/components/transforms/base/log_to_metric.cue Co-authored-by: Heston Hoffman Update src/transforms/log_to_metric.rs Co-authored-by: Heston Hoffman Update src/transforms/log_to_metric.rs Co-authored-by: Heston Hoffman Update src/transforms/log_to_metric.rs Co-authored-by: Heston Hoffman Fix styling in example for all_metrics. Minor tweaks to formatting for documentation Fix minor issues with formatting that was causing tests to fail. --- .github/workflows/create_preview_sites.yml | 7 +- .github/workflows/protobuf.yml | 2 +- Cargo.lock | 45 +- Cargo.toml | 5 +- README.md | 4 +- docs/DEPRECATIONS.md | 2 - lib/vector-api-client/Cargo.toml | 2 +- lib/vector-buffers/Cargo.toml | 6 +- lib/vector-config/src/external/datetime.rs | 4 +- lib/vector-core/Cargo.toml | 4 +- lib/vector-stream/Cargo.toml | 2 +- lib/vector-vrl/cli/Cargo.toml | 2 +- lib/vector-vrl/tests/Cargo.toml | 2 +- lib/vector-vrl/web-playground/build.rs | 72 +- lib/vector-vrl/web-playground/public/index.js | 16 +- lib/vector-vrl/web-playground/src/lib.rs | 9 + src/components/validation/runner/config.rs | 2 +- src/components/validation/runner/telemetry.rs | 2 +- src/internal_events/file.rs | 8 +- src/internal_events/log_to_metric.rs | 65 +- src/sinks/appsignal/config.rs | 2 +- src/sinks/aws_cloudwatch_logs/config.rs | 6 +- src/sinks/aws_cloudwatch_logs/service.rs | 9 +- src/sinks/aws_cloudwatch_metrics/mod.rs | 17 +- src/sinks/aws_kinesis/config.rs | 2 +- .../aws_kinesis/firehose/integration_tests.rs | 4 +- src/sinks/aws_s3/config.rs | 33 +- src/sinks/aws_s3/integration_tests.rs | 20 +- src/sinks/aws_s3/mod.rs | 2 +- src/sinks/aws_s3/sink.rs | 12 +- src/sinks/aws_s_s/sink.rs | 4 +- src/sinks/azure_blob/config.rs | 14 +- src/sinks/azure_monitor_logs/config.rs | 2 +- src/sinks/clickhouse/config.rs | 2 +- src/sinks/clickhouse/integration_tests.rs | 6 +- src/sinks/databend/config.rs | 2 +- src/sinks/datadog/events/config.rs | 2 +- src/sinks/datadog/logs/config.rs | 2 +- src/sinks/datadog/metrics/config.rs | 19 +- src/sinks/datadog/traces/config.rs | 9 +- src/sinks/elasticsearch/common.rs | 7 +- src/sinks/elasticsearch/config.rs | 7 +- src/sinks/file/mod.rs | 52 +- src/sinks/gcp/chronicle_unstructured.rs | 14 +- src/sinks/gcp/cloud_storage.rs | 63 +- src/sinks/gcp/pubsub.rs | 2 +- src/sinks/gcp/stackdriver/logs/config.rs | 16 +- src/sinks/gcp/stackdriver/metrics/config.rs | 20 +- src/sinks/greptimedb/mod.rs | 2 +- src/sinks/honeycomb/config.rs | 2 +- src/sinks/http/config.rs | 2 +- src/sinks/influxdb/logs.rs | 5 +- src/sinks/influxdb/metrics.rs | 5 +- src/sinks/loki/sink.rs | 4 +- src/sinks/mezmo.rs | 2 +- src/sinks/nats/config.rs | 11 +- src/sinks/nats/sink.rs | 9 +- src/sinks/new_relic/config.rs | 2 +- src/sinks/prometheus/remote_write/config.rs | 2 +- src/sinks/redis/config.rs | 11 +- src/sinks/redis/integration_tests.rs | 8 +- src/sinks/redis/sink.rs | 14 +- src/sinks/sematext/metrics.rs | 5 +- src/sinks/splunk_hec/logs/config.rs | 2 +- src/sinks/splunk_hec/metrics/config.rs | 2 +- .../util/adaptive_concurrency/controller.rs | 4 +- src/sinks/util/adaptive_concurrency/mod.rs | 20 +- src/sinks/util/adaptive_concurrency/tests.rs | 11 +- src/sinks/util/mod.rs | 10 +- src/sinks/util/retries.rs | 117 +++- src/sinks/util/service.rs | 297 ++++---- src/sinks/util/service/concurrency.rs | 15 +- src/sinks/vector/config.rs | 2 +- src/sources/datadog_agent/mod.rs | 8 +- src/sources/file.rs | 3 + src/sources/kafka.rs | 2 +- src/template.rs | 60 +- src/transforms/aggregate.rs | 2 +- src/transforms/log_to_metric.rs | 654 +++++++++++------- vdev/Cargo.toml | 2 +- .../installation/package-managers/apt.md | 2 +- .../installation/package-managers/yum.md | 2 +- .../highlights/2023-11-07-new-linux-repos.md | 2 +- .../2023-12-19-0-35-0-upgrade-guide.md | 32 +- .../components/sinks/base/appsignal.cue | 43 +- .../sinks/base/aws_cloudwatch_logs.cue | 39 +- .../sinks/base/aws_cloudwatch_metrics.cue | 45 +- .../sinks/base/aws_kinesis_firehose.cue | 43 +- .../sinks/base/aws_kinesis_streams.cue | 43 +- .../components/sinks/base/aws_s3.cue | 54 +- .../components/sinks/base/aws_sns.cue | 43 +- .../components/sinks/base/aws_sqs.cue | 43 +- .../reference/components/sinks/base/axiom.cue | 39 +- .../components/sinks/base/azure_blob.cue | 45 +- .../sinks/base/azure_monitor_logs.cue | 43 +- .../components/sinks/base/clickhouse.cue | 43 +- .../components/sinks/base/databend.cue | 43 +- .../components/sinks/base/datadog_events.cue | 43 +- .../components/sinks/base/datadog_logs.cue | 39 +- .../components/sinks/base/datadog_metrics.cue | 43 +- .../components/sinks/base/datadog_traces.cue | 43 +- .../components/sinks/base/elasticsearch.cue | 39 +- .../reference/components/sinks/base/file.cue | 15 +- .../sinks/base/gcp_chronicle_unstructured.cue | 45 +- .../sinks/base/gcp_cloud_storage.cue | 56 +- .../components/sinks/base/gcp_pubsub.cue | 43 +- .../sinks/base/gcp_stackdriver_logs.cue | 45 +- .../sinks/base/gcp_stackdriver_metrics.cue | 45 +- .../components/sinks/base/greptimedb.cue | 43 +- .../components/sinks/base/honeycomb.cue | 43 +- .../reference/components/sinks/base/http.cue | 39 +- .../components/sinks/base/humio_logs.cue | 43 +- .../components/sinks/base/humio_metrics.cue | 43 +- .../components/sinks/base/influxdb_logs.cue | 43 +- .../sinks/base/influxdb_metrics.cue | 43 +- .../components/sinks/base/logdna.cue | 43 +- .../reference/components/sinks/base/loki.cue | 43 +- .../reference/components/sinks/base/mezmo.cue | 43 +- .../reference/components/sinks/base/nats.cue | 45 +- .../components/sinks/base/new_relic.cue | 43 +- .../sinks/base/prometheus_remote_write.cue | 43 +- .../reference/components/sinks/base/redis.cue | 45 +- .../components/sinks/base/sematext_logs.cue | 43 +- .../sinks/base/sematext_metrics.cue | 43 +- .../components/sinks/base/splunk_hec_logs.cue | 43 +- .../sinks/base/splunk_hec_metrics.cue | 43 +- .../components/sinks/base/vector.cue | 43 +- .../components/sources/base/datadog_agent.cue | 8 +- .../components/sources/base/file.cue | 4 +- .../components/sources/base/kafka.cue | 2 +- .../sources/base/kubernetes_logs.cue | 4 +- .../components/transforms/base/aggregate.cue | 2 +- .../transforms/base/log_to_metric.cue | 158 +++-- website/cue/reference/releases/0.34.0.cue | 9 + 134 files changed, 2767 insertions(+), 1203 deletions(-) diff --git a/.github/workflows/create_preview_sites.yml b/.github/workflows/create_preview_sites.yml index 40dd37731cba09..dc938eba0f838f 100644 --- a/.github/workflows/create_preview_sites.yml +++ b/.github/workflows/create_preview_sites.yml @@ -52,8 +52,9 @@ jobs: run: | unzip pr.zip -d pr BRANCH_NAME=$(cat ./pr/branch) - SANITIZED_BRANCH_NAME=$(echo "$BRANCH_NAME" | sed 's/\./-/g') + SANITIZED_BRANCH_NAME=$(echo "$BRANCH_NAME" | sed 's/[\/\.]/-/g') echo "SANITIZED_BRANCH_NAME=$SANITIZED_BRANCH_NAME" >> $GITHUB_ENV + echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV # Kick off the job in amplify - name: Deploy Site @@ -64,12 +65,13 @@ jobs: REQUEST_MESSAGE: ${{ secrets.REQUEST_MESSAGE }} ENDPOINT: ${{ secrets.ENDPOINT }} run: | + sleep 20 HMAC_KEY=$(echo -n $REQUEST_MESSAGE | openssl dgst -sha256 -hmac "$REQUEST_TOKEN" -binary | od -An -tx1 | tr -d ' \n'; echo) SIGNATURE="sha256=$HMAC_KEY" RESPONSE_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ -H "Content-Type: application/json" \ -H "X-Hub-Signature: $SIGNATURE" \ - -d "{\"app_id\": \"$APP_ID\", \"branch_name\": \"$SANITIZED_BRANCH_NAME\"}" \ + -d "{\"app_id\": \"$APP_ID\", \"branch_name\": \"$BRANCH_NAME\"}" \ "$ENDPOINT") # check the response code and fail if not 200 @@ -91,6 +93,7 @@ jobs: const fs = require('fs'); const prNumber = fs.readFileSync('./pr/number', 'utf8'); const issueNumber = parseInt(prNumber); + const { APP_ID, APP_NAME, SANITIZED_BRANCH_NAME } = process.env await github.rest.issues.createComment({ owner: context.repo.owner, diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index 85c3093f228c55..5679b441d8baa4 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -20,7 +20,7 @@ jobs: # Run `git checkout` - uses: actions/checkout@v3 # Install the `buf` CLI - - uses: bufbuild/buf-setup-action@v1.27.2 + - uses: bufbuild/buf-setup-action@v1.28.0 # Perform breaking change detection against the `master` branch - uses: bufbuild/buf-breaking-action@v1.1.3 with: diff --git a/Cargo.lock b/Cargo.lock index 53d6169912249b..136fe6d8e62232 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2041,9 +2041,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.7" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" +checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" dependencies = [ "clap_builder", "clap_derive", @@ -2055,15 +2055,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5fdbb015d790cfb378aca82caf9cc52a38be96a7eecdb92f31b4366a8afc019" dependencies = [ - "clap 4.4.7", + "clap 4.4.8", "log", ] [[package]] name = "clap_builder" -version = "4.4.7" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" +checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" dependencies = [ "anstream", "anstyle", @@ -2078,7 +2078,7 @@ version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bffe91f06a11b4b9420f62103854e90867812cd5d01557f853c5ee8e791b12ae" dependencies = [ - "clap 4.4.7", + "clap 4.4.8", ] [[package]] @@ -2403,7 +2403,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.7", + "clap 4.4.8", "criterion-plot", "futures 0.3.29", "is-terminal", @@ -3174,9 +3174,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" dependencies = [ "humantime", "is-terminal", @@ -3870,9 +3870,9 @@ dependencies = [ [[package]] name = "hdrhistogram" -version = "7.5.2" +version = "7.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" +checksum = "a5b38e5c02b7c7be48c8dc5217c4f1634af2ea221caae2e024bffc7a7651c691" dependencies = [ "base64 0.13.1", "byteorder", @@ -4637,7 +4637,7 @@ dependencies = [ name = "k8s-e2e-tests" version = "0.1.0" dependencies = [ - "env_logger 0.10.0", + "env_logger 0.10.1", "futures 0.3.29", "indoc", "k8s-openapi 0.16.0", @@ -6743,9 +6743,9 @@ dependencies = [ [[package]] name = "proptest" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c003ac8c77cb07bb74f5f198bce836a689bcd5a42574612bf14d17bfd08c20e" +checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" dependencies = [ "bit-set", "bit-vec", @@ -6755,7 +6755,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", "rusty-fork", "tempfile", "unarray", @@ -9840,7 +9840,7 @@ dependencies = [ "anyhow", "cached", "chrono", - "clap 4.4.7", + "clap 4.4.8", "clap-verbosity-flag", "clap_complete", "confy", @@ -9915,8 +9915,9 @@ dependencies = [ "bytes 1.5.0", "bytesize", "chrono", + "chrono-tz", "cidr-utils", - "clap 4.4.7", + "clap 4.4.8", "colored", "console-subscriber", "criterion", @@ -10065,7 +10066,7 @@ dependencies = [ "anyhow", "async-trait", "chrono", - "clap 4.4.7", + "clap 4.4.8", "futures 0.3.29", "graphql_client", "indoc", @@ -10088,7 +10089,7 @@ dependencies = [ "async-trait", "bytecheck", "bytes 1.5.0", - "clap 4.4.7", + "clap 4.4.8", "crc32fast", "criterion", "crossbeam-queue", @@ -10344,7 +10345,7 @@ dependencies = [ name = "vector-vrl-cli" version = "0.1.0" dependencies = [ - "clap 4.4.7", + "clap 4.4.8", "vector-vrl-functions", "vrl", ] @@ -10363,7 +10364,7 @@ dependencies = [ "ansi_term", "chrono", "chrono-tz", - "clap 4.4.7", + "clap 4.4.8", "enrichment", "glob", "prettydiff", @@ -10423,7 +10424,7 @@ dependencies = [ "chrono", "chrono-tz", "cidr-utils", - "clap 4.4.7", + "clap 4.4.8", "codespan-reporting", "community-id", "crypto_secretbox", diff --git a/Cargo.toml b/Cargo.toml index 35670f82ee96a9..d92e02860bf38c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -242,8 +242,9 @@ bollard = { version = "0.15.0", default-features = false, features = ["ssl", "ch bytes = { version = "1.5.0", default-features = false, features = ["serde"] } bytesize = { version = "1.3.0", default-features = false } chrono = { version = "0.4.31", default-features = false, features = ["serde"] } +chrono-tz = { version = "0.8.3", default-features = false } cidr-utils = { version = "0.5.11", default-features = false } -clap = { version = "4.4.7", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } +clap = { version = "4.4.8", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } colored = { version = "2.0.4", default-features = false } csv = { version = "1.3", default-features = false } derivative = { version = "2.2.0", default-features = false } @@ -351,7 +352,7 @@ criterion = { version = "0.5.1", features = ["html_reports", "async_tokio"] } itertools = { version = "0.11.0", default-features = false, features = ["use_alloc"] } libc = "0.2.150" similar-asserts = "1.5.0" -proptest = "1.3" +proptest = "1.4" quickcheck = "1.0.3" reqwest = { version = "0.11", features = ["json"] } rstest = {version = "0.18.2"} diff --git a/README.md b/README.md index 1ab1fb21812a21..2b0c18f6741477 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Vector is a high-performance, end-to-end (agent & aggregator) observability data pipeline that puts you in control of your observability data. [Collect][docs.sources], [transform][docs.transforms], and [route][docs.sinks] -all your logs, metrics, and traces to any vendors you want today and any other +all your logs and metrics to any vendors you want today and any other vendors you may want tomorrow. Vector enables dramatic cost reduction, novel data enrichment, and data security where you need it, not where it is most convenient for your vendors. Additionally, it is open source and up to 10x @@ -31,7 +31,7 @@ Vector**][docs.installation]. * **Reliable** - Built in [Rust][urls.rust], Vector's primary design goal is reliability. * **End-to-end** - Deploys as an [agent][docs.roles#agent] or [aggregator][docs.roles#aggregator]. Vector is a complete platform. -* **Unified** - [Logs][docs.data-model.log], [metrics][docs.data-model.metric], and traces (coming soon). One tool for all of your data. +* **Unified** - [Logs][docs.data-model.log], [metrics][docs.data-model.metric] (beta), and traces (coming soon). One tool for all of your data. ### Use cases diff --git a/docs/DEPRECATIONS.md b/docs/DEPRECATIONS.md index ad3a1d683b6a25..b2e1faedd69962 100644 --- a/docs/DEPRECATIONS.md +++ b/docs/DEPRECATIONS.md @@ -16,8 +16,6 @@ For example: ## To be migrated -- file_metric_tag v0.36.0 The `internal_metrics.include_file_tag` should default to `false`. - ## To be removed - datadog_v1_metrics v0.35.0 Support for `v1` series endpoint in the `datadog_metrics` sink should be removed. diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 2a03658c8173ca..c121364762b885 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -30,7 +30,7 @@ tokio-tungstenite = { version = "0.20.1", default-features = false, features = [ # External libs chrono = { version = "0.4.31", default-features = false, features = ["serde"] } -clap = { version = "4.4.7", default-features = false, features = ["derive"] } +clap = { version = "4.4.8", default-features = false, features = ["derive"] } url = { version = "2.4.1", default-features = false } uuid = { version = "1", default-features = false, features = ["serde", "v4"] } indoc = { version = "2.0.4", default-features = false } diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index b3f36267463d42..24dacd08ee7e66 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -34,14 +34,14 @@ vector-config-macros = { path = "../vector-config-macros", default-features = fa vector-common = { path = "../vector-common", default-features = false, features = ["byte_size_of", "serde"] } [dev-dependencies] -clap = "4.4.7" +clap = "4.4.8" criterion = { version = "0.5", features = ["html_reports", "async_tokio"] } crossbeam-queue = "0.3.8" -hdrhistogram = "7.5.2" +hdrhistogram = "7.5.3" metrics-tracing-context = { version = "0.14.0", default-features = false } metrics-util = { version = "0.15.1", default-features = false, features = ["debugging"] } once_cell = "1.18" -proptest = "1.3" +proptest = "1.4" quickcheck = "1.0" rand = "0.8.5" serde_yaml = { version = "0.9", default-features = false } diff --git a/lib/vector-config/src/external/datetime.rs b/lib/vector-config/src/external/datetime.rs index 45575f28465649..6735cfa2e239c4 100644 --- a/lib/vector-config/src/external/datetime.rs +++ b/lib/vector-config/src/external/datetime.rs @@ -20,8 +20,8 @@ impl Configurable for TimeZone { fn metadata() -> Metadata { let mut metadata = Metadata::default(); - metadata.set_title("Timezone reference."); - metadata.set_description(r#"This can refer to any valid timezone as defined in the [TZ database][tzdb], or "local" which refers to the system local timezone. + metadata.set_title("Timezone to use for any date specifiers in template strings."); + metadata.set_description(r#"This can refer to any valid timezone as defined in the [TZ database][tzdb], or "local" which refers to the system local timezone. It will default to the [globally configured timezone](https://vector.dev/docs/reference/configuration/global-options/#timezone). [tzdb]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones"#); metadata.add_custom_attribute(CustomAttribute::kv( diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index c62e3910561f8d..ce276db48b219b 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -34,7 +34,7 @@ ordered-float = { version = "4.1.1", default-features = false } openssl = { version = "0.10.59", default-features = false, features = ["vendored"] } parking_lot = { version = "0.12.1", default-features = false } pin-project.workspace = true -proptest = { version = "1.3", optional = true } +proptest = { version = "1.4", optional = true } prost-types = { version = "0.12", default-features = false } prost = { version = "0.12", default-features = false, features = ["std"] } quanta = { version = "0.12.1", default-features = false } @@ -80,7 +80,7 @@ criterion = { version = "0.5.1", features = ["html_reports"] } env-test-util = "1.0.1" quickcheck = "1" quickcheck_macros = "1" -proptest = "1.3" +proptest = "1.4" similar-asserts = "1.5.0" tokio-test = "0.4.3" toml = { version = "0.8.8", default-features = false, features = ["parse"] } diff --git a/lib/vector-stream/Cargo.toml b/lib/vector-stream/Cargo.toml index 5525a4d7532216..794c03acde6aed 100644 --- a/lib/vector-stream/Cargo.toml +++ b/lib/vector-stream/Cargo.toml @@ -19,6 +19,6 @@ vector-common = { path = "../vector-common" } vector-core = { path = "../vector-core" } [dev-dependencies] -proptest = "1.3" +proptest = "1.4" rand = "0.8.5" rand_distr = "0.4.3" diff --git a/lib/vector-vrl/cli/Cargo.toml b/lib/vector-vrl/cli/Cargo.toml index e374afc1894c30..3f0b081a02aa3e 100644 --- a/lib/vector-vrl/cli/Cargo.toml +++ b/lib/vector-vrl/cli/Cargo.toml @@ -7,6 +7,6 @@ publish = false license = "MPL-2.0" [dependencies] -clap = { version = "4.4.7", features = ["derive"] } +clap = { version = "4.4.8", features = ["derive"] } vector-vrl-functions = { path = "../functions" } vrl.workspace = true diff --git a/lib/vector-vrl/tests/Cargo.toml b/lib/vector-vrl/tests/Cargo.toml index 72aeae3861191e..f2684e1c003fc6 100644 --- a/lib/vector-vrl/tests/Cargo.toml +++ b/lib/vector-vrl/tests/Cargo.toml @@ -13,7 +13,7 @@ vector-vrl-functions = { path = "../../vector-vrl/functions" } ansi_term = "0.12" chrono = "0.4" chrono-tz = "0.8" -clap = { version = "4.4.7", features = ["derive"] } +clap = { version = "4.4.8", features = ["derive"] } glob = "0.3" prettydiff = "0.6" regex = "1" diff --git a/lib/vector-vrl/web-playground/build.rs b/lib/vector-vrl/web-playground/build.rs index f6d42756f10577..d6abdbbd3090e2 100644 --- a/lib/vector-vrl/web-playground/build.rs +++ b/lib/vector-vrl/web-playground/build.rs @@ -24,40 +24,68 @@ fn get_git_hash() -> String { .args(["rev-parse", "HEAD"]) .output() .expect("Failed to get git HEAD sha"); - let mut git_hash = String::from_utf8(output.stdout).unwrap(); - git_hash.truncate(8); - git_hash + String::from_utf8(output.stdout).unwrap().trim().to_string() } -fn write_build_constants(manifest: &Manifest, dest_path: &Path) -> io::Result<()> { - let mut output_file = File::create(dest_path)?; - output_file.write_all( - "// AUTOGENERATED CONSTANTS. SEE BUILD.RS AT REPOSITORY ROOT. DO NOT MODIFY.\n".as_ref(), - )?; - - let create_const_statement = - |name, value| format!("pub const {}: &str = \"{}\";\n", name, value); +fn create_const_statement(name: &str, value: &str) -> String { + format!("pub const {name}: &str = \"{value}\";\n") +} +fn write_vector_constants(output_file: &mut File) { // TODO: For releases, we should use the manifest.package().version(). // https://github.com/vectordotdev/vector/issues/18425 - let vector_version_const = create_const_statement("VECTOR_VERSION", get_git_hash()); + let vector_git_sha = get_git_hash(); + let vector_version_statement = create_const_statement("VECTOR_VERSION", &vector_git_sha); + output_file + .write_all(vector_version_statement.as_bytes()) + .expect("Failed to write Vector version constant"); + + let vector_link = format!("https://github.com/vectordotdev/vector/tree/{vector_git_sha}"); + let vector_link_statement = create_const_statement("VECTOR_LINK", &vector_link); output_file - .write_all(vector_version_const.as_bytes()) + .write_all(vector_link_statement.as_bytes()) .expect("Failed to write Vector version constant"); +} - let vrl_version = manifest +fn write_vrl_constants(manifest: &Manifest, output_file: &mut File) { + let vrl_dep = manifest .dependencies .get("vrl") - .unwrap() + .expect("missing VRL dependency") .detail() - .unwrap() - .version - .clone() - .unwrap_or_else(|| "FIXME".into()); - let vrl_version_const = create_const_statement("VRL_VERSION", vrl_version); + .expect("expected detail dependency format"); + + let (version, link) = match &vrl_dep.version { + None => { + let repo = vrl_dep + .git + .as_ref() + .expect("VRL dependency should use 'version' or 'git'"); + let version = vrl_dep + .rev + .as_ref() + .expect("VRL git revision not specified"); + (version.clone(), format!("{repo}/tree/{version}")) + } + Some(v) => (v.clone(), format!("https://crates.io/crates/vrl/{v}")), + }; + output_file - .write_all(vrl_version_const.as_bytes()) - .expect("Failed to write Vector version constant"); + .write_all(create_const_statement("VRL_VERSION", &version).as_bytes()) + .expect("Failed to write VRL version constant"); + + output_file + .write_all(create_const_statement("VRL_LINK", &link).as_bytes()) + .expect("Failed to write VRL_LINK constant"); +} + +fn write_build_constants(manifest: &Manifest, dest_path: &Path) -> io::Result<()> { + let mut output_file = File::create(dest_path)?; + output_file.write_all( + "// AUTOGENERATED CONSTANTS. SEE BUILD.RS AT REPOSITORY ROOT. DO NOT MODIFY.\n".as_ref(), + )?; + write_vector_constants(&mut output_file); + write_vrl_constants(manifest, &mut output_file); Ok(()) } diff --git a/lib/vector-vrl/web-playground/public/index.js b/lib/vector-vrl/web-playground/public/index.js index 377167f8f6cc90..aaf0c870072538 100644 --- a/lib/vector-vrl/web-playground/public/index.js +++ b/lib/vector-vrl/web-playground/public/index.js @@ -1,4 +1,4 @@ -import init, { run_vrl, vrl_version, vector_version } from "./pkg/vector_vrl_web_playground.js"; +import init, { run_vrl, vrl_version, vrl_link, vector_version, vector_link } from "./pkg/vector_vrl_web_playground.js"; import { vrlLanguageDefinition, vrlThemeDefinition } from "./vrl-highlighter.js"; const PROGRAM_EDITOR_DEFAULT_VALUE = `# Remove some fields @@ -39,8 +39,12 @@ export class VrlWebPlayground { constructor() { let temp = init().then(() => { this.run_vrl = run_vrl; + this.vector_version = vector_version(); + this.vector_link = vector_link(); + this.vrl_version = vrl_version(); + this.vrl_link = vrl_link(); // require is provided by loader.min.js. require.config({ @@ -92,14 +96,12 @@ export class VrlWebPlayground { addVersions() { let vectorLinkElement = document.getElementById('vector-version-link'); - let vectorVersion = vector_version(); - vectorLinkElement.text = vectorVersion; - vectorLinkElement.href = `https://github.com/vectordotdev/vector/tree/${vectorVersion}`; + vectorLinkElement.text = this.vector_version.substring(0, 8); + vectorLinkElement.href = this.vector_link; let vrlLinkElement = document.getElementById('vrl-version-link'); - let vrlVersion = vrl_version(); - vrlLinkElement.text = vrlVersion; - vrlLinkElement.href = `https://crates.io/crates/vrl/${vrlVersion}`; + vrlLinkElement.text = this.vrl_version.substring(0, 8); + vrlLinkElement.href = this.vrl_link; } createDefaultEditor(elementId, value, language, theme) { diff --git a/lib/vector-vrl/web-playground/src/lib.rs b/lib/vector-vrl/web-playground/src/lib.rs index 51a23cacfbd9b1..a434050aefa3c2 100644 --- a/lib/vector-vrl/web-playground/src/lib.rs +++ b/lib/vector-vrl/web-playground/src/lib.rs @@ -120,7 +120,16 @@ pub fn vector_version() -> String { built_info::VECTOR_VERSION.to_string() } +#[wasm_bindgen] +pub fn vector_link() -> String { + built_info::VECTOR_LINK.to_string() +} + #[wasm_bindgen] pub fn vrl_version() -> String { built_info::VRL_VERSION.to_string() } +#[wasm_bindgen] +pub fn vrl_link() -> String { + built_info::VRL_LINK.to_string() +} diff --git a/src/components/validation/runner/config.rs b/src/components/validation/runner/config.rs index c087fd8bf809aa..f80746ab21df19 100644 --- a/src/components/validation/runner/config.rs +++ b/src/components/validation/runner/config.rs @@ -141,7 +141,7 @@ fn build_output_edge() -> (OutputEdge, impl Into) { // we don't want to waste time performing retries, especially when the test // harness is shutting down. output_sink.batch.timeout_secs = Some(0.1); - output_sink.request.retry_attempts = Some(0); + output_sink.request.retry_attempts = 0; let output_edge = OutputEdge::from_address(output_listen_addr); diff --git a/src/components/validation/runner/telemetry.rs b/src/components/validation/runner/telemetry.rs index d5bb8f5b438d3d..c415bfb05d0127 100644 --- a/src/components/validation/runner/telemetry.rs +++ b/src/components/validation/runner/telemetry.rs @@ -52,7 +52,7 @@ impl Telemetry { // disable retries, as we don't want to waste time performing retries, // especially when the test harness is shutting down. vector_sink.batch.timeout_secs = Some(0.1); - vector_sink.request.retry_attempts = Some(0); + vector_sink.request.retry_attempts = 0; config_builder.add_source(INTERNAL_LOGS_KEY, internal_logs); config_builder.add_source(INTERNAL_METRICS_KEY, internal_metrics); diff --git a/src/internal_events/file.rs b/src/internal_events/file.rs index 15370ad9cdbd20..fc493a1a83c787 100644 --- a/src/internal_events/file.rs +++ b/src/internal_events/file.rs @@ -12,16 +12,14 @@ use vector_lib::internal_event::{error_stage, error_type}; /// Configuration of internal metrics for file-based components. #[configurable_component] -#[derive(Clone, Debug, PartialEq, Eq, Derivative)] -#[derivative(Default)] +#[derive(Clone, Debug, PartialEq, Eq, Default)] #[serde(deny_unknown_fields)] pub struct FileInternalMetricsConfig { /// Whether or not to include the "file" tag on the component's corresponding internal metrics. /// /// This is useful for distinguishing between different files while monitoring. However, the tag's - /// cardinality is unbounded. This defaults to true for now but will default to false in a future version. - #[serde(default = "crate::serde::default_true")] - #[derivative(Default(value = "true"))] + /// cardinality is unbounded. + #[serde(default = "crate::serde::default_false")] pub include_file_tag: bool, } diff --git a/src/internal_events/log_to_metric.rs b/src/internal_events/log_to_metric.rs index db3b5d5bca5500..68bad4df8af1ca 100644 --- a/src/internal_events/log_to_metric.rs +++ b/src/internal_events/log_to_metric.rs @@ -90,51 +90,25 @@ impl<'a> InternalEvent for MetricMetadataInvalidFieldValueError<'a> { } } -pub struct MetricMetadataParseFloatError<'a> { +pub struct MetricMetadataParseError<'a> { pub field: &'a str, + pub kind: &'a str, } -impl<'a> InternalEvent for MetricMetadataParseFloatError<'a> { +impl<'a> InternalEvent for MetricMetadataParseError<'a> { fn emit(self) { let reason = "Failed to parse field as float."; error!( message = reason, field = %self.field, - error_code = "failed_parsing_float", - error_type = error_type::PARSER_FAILED, - stage = error_stage::PROCESSING, - internal_log_rate_limit = true - ); - counter!( - "component_errors_total", 1, - "error_code" => "failed_parsing_float", - "error_type" => error_type::PARSER_FAILED, - "stage" => error_stage::PROCESSING, - "field" => self.field.to_string(), - ); - - emit!(ComponentEventsDropped:: { count: 1, reason }) - } -} - -pub struct MetricMetadataParseIntError<'a> { - pub field: &'a str, -} - -impl<'a> InternalEvent for MetricMetadataParseIntError<'a> { - fn emit(self) { - let reason = "Failed to parse field as int."; - error!( - message = reason, - field = %self.field, - error_code = "failed_parsing_int", + error_code = format!("failed_parsing_{}", self.kind), error_type = error_type::PARSER_FAILED, stage = error_stage::PROCESSING, internal_log_rate_limit = true ); counter!( "component_errors_total", 1, - "error_code" => "failed_parsing_int", + "error_code" => format!("failed_parsing_{}", self.kind), "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, "field" => self.field.to_string(), @@ -144,34 +118,7 @@ impl<'a> InternalEvent for MetricMetadataParseIntError<'a> { } } -pub struct MetricMetadataParseArrayError<'a> { - pub field: &'a str, -} - -impl<'a> InternalEvent for MetricMetadataParseArrayError<'a> { - fn emit(self) { - let reason = "Failed to parse field as array."; - error!( - message = reason, - field = %self.field, - error_code = "failed_parsing_array", - error_type = error_type::PARSER_FAILED, - stage = error_stage::PROCESSING, - internal_log_rate_limit = true - ); - counter!( - "component_errors_total", 1, - "error_code" => "failed_parsing_array", - "error_type" => error_type::PARSER_FAILED, - "stage" => error_stage::PROCESSING, - "field" => self.field.to_string(), - ); - - emit!(ComponentEventsDropped:: { count: 1, reason }) - } -} -pub struct MetricMetadataMetricDetailsNotFoundError { -} +pub struct MetricMetadataMetricDetailsNotFoundError {} impl InternalEvent for MetricMetadataMetricDetailsNotFoundError { fn emit(self) { diff --git a/src/sinks/appsignal/config.rs b/src/sinks/appsignal/config.rs index 2fc0bdb4907b5e..137757f3a7fc89 100644 --- a/src/sinks/appsignal/config.rs +++ b/src/sinks/appsignal/config.rs @@ -102,7 +102,7 @@ impl AppsignalConfig { let service = AppsignalService::new(http_client, endpoint, push_api_key, compression); let request_opts = self.request; - let request_settings = request_opts.unwrap_with(&TowerRequestConfig::default()); + let request_settings = request_opts.into_settings(); let retry_logic = HttpStatusRetryLogic::new(|req: &AppsignalResponse| req.http_status); let service = ServiceBuilder::new() diff --git a/src/sinks/aws_cloudwatch_logs/config.rs b/src/sinks/aws_cloudwatch_logs/config.rs index 8b8b94662376c9..18e9201421c6f6 100644 --- a/src/sinks/aws_cloudwatch_logs/config.rs +++ b/src/sinks/aws_cloudwatch_logs/config.rs @@ -25,7 +25,6 @@ use crate::{ }, util::{ http::RequestConfig, BatchConfig, Compression, ServiceBuilderExt, SinkBatchSettings, - TowerRequestConfig, }, Healthcheck, VectorSink, }, @@ -211,10 +210,7 @@ impl CloudwatchLogsSinkConfig { impl SinkConfig for CloudwatchLogsSinkConfig { async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { let batcher_settings = self.batch.into_batcher_settings()?; - let request_settings = self - .request - .tower - .unwrap_with(&TowerRequestConfig::default()); + let request_settings = self.request.tower.into_settings(); let client = self.create_client(cx.proxy()).await?; let smithy_client = self.create_smithy_client(cx.proxy()).await?; let svc = ServiceBuilder::new() diff --git a/src/sinks/aws_cloudwatch_logs/service.rs b/src/sinks/aws_cloudwatch_logs/service.rs index b3703cf9166a83..bedfc0d3574cb4 100644 --- a/src/sinks/aws_cloudwatch_logs/service.rs +++ b/src/sinks/aws_cloudwatch_logs/service.rs @@ -34,14 +34,14 @@ use crate::sinks::{ config::CloudwatchLogsSinkConfig, config::Retention, request, retry::CloudwatchRetryLogic, sink::BatchCloudwatchRequest, CloudwatchKey, }, - util::{retries::FixedRetryPolicy, EncodedLength, TowerRequestConfig, TowerRequestSettings}, + util::{retries::FibonacciRetryPolicy, EncodedLength, TowerRequestSettings}, }; type Svc = Buffer< ConcurrencyLimit< RateLimit< Retry< - FixedRetryPolicy>, + FibonacciRetryPolicy>, Buffer, Vec>, >, >, @@ -136,10 +136,7 @@ impl CloudwatchLogsPartitionSvc { // https://github.com/awslabs/aws-sdk-rust/issues/537 smithy_client: SmithyClient, ) -> Self { - let request_settings = config - .request - .tower - .unwrap_with(&TowerRequestConfig::default()); + let request_settings = config.request.tower.into_settings(); Self { config, diff --git a/src/sinks/aws_cloudwatch_metrics/mod.rs b/src/sinks/aws_cloudwatch_metrics/mod.rs index af81043a29bdbe..8032aa41d0e5be 100644 --- a/src/sinks/aws_cloudwatch_metrics/mod.rs +++ b/src/sinks/aws_cloudwatch_metrics/mod.rs @@ -36,6 +36,8 @@ use crate::{ tls::TlsConfig, }; +use super::util::service::TowerRequestConfigDefaults; + #[derive(Clone, Copy, Debug, Default)] pub struct CloudWatchMetricsDefaultBatchSettings; @@ -45,6 +47,13 @@ impl SinkBatchSettings for CloudWatchMetricsDefaultBatchSettings { const TIMEOUT_SECS: f64 = 1.0; } +#[derive(Clone, Copy, Debug)] +pub struct CloudWatchMetricsTowerRequestConfigDefaults; + +impl TowerRequestConfigDefaults for CloudWatchMetricsTowerRequestConfigDefaults { + const RATE_LIMIT_NUM: u64 = 150; +} + /// Configuration for the `aws_cloudwatch_metrics` sink. #[configurable_component(sink( "aws_cloudwatch_metrics", @@ -79,7 +88,7 @@ pub struct CloudWatchMetricsSinkConfig { #[configurable(derived)] #[serde(default)] - pub request: TowerRequestConfig, + pub request: TowerRequestConfig, #[configurable(derived)] pub tls: Option, @@ -225,11 +234,7 @@ impl CloudWatchMetricsSvc { ) -> crate::Result { let default_namespace = config.default_namespace.clone(); let batch = config.batch.into_batch_settings()?; - let request_settings = config.request.unwrap_with( - &TowerRequestConfig::default() - .timeout_secs(30) - .rate_limit_num(150), - ); + let request_settings = config.request.into_settings(); let service = CloudWatchMetricsSvc { client }; let buffer = PartitionBuffer::new(MetricsBuffer::new(batch.size)); diff --git a/src/sinks/aws_kinesis/config.rs b/src/sinks/aws_kinesis/config.rs index 341063a0f2f191..d5356efc9fbbb9 100644 --- a/src/sinks/aws_kinesis/config.rs +++ b/src/sinks/aws_kinesis/config.rs @@ -95,7 +95,7 @@ where E: Send + 'static, RT: RetryLogic + Default, { - let request_limits = config.request.unwrap_with(&TowerRequestConfig::default()); + let request_limits = config.request.into_settings(); let region = config.region.region(); let service = ServiceBuilder::new() diff --git a/src/sinks/aws_kinesis/firehose/integration_tests.rs b/src/sinks/aws_kinesis/firehose/integration_tests.rs index 002df851b1085d..7ce6ab05beb2f8 100644 --- a/src/sinks/aws_kinesis/firehose/integration_tests.rs +++ b/src/sinks/aws_kinesis/firehose/integration_tests.rs @@ -52,8 +52,8 @@ async fn firehose_put_records() { encoding: JsonSerializerConfig::default().into(), // required for ES destination w/ localstack compression: Compression::None, request: TowerRequestConfig { - timeout_secs: Some(10), - retry_attempts: Some(0), + timeout_secs: 10, + retry_attempts: 0, ..Default::default() }, tls: None, diff --git a/src/sinks/aws_s3/config.rs b/src/sinks/aws_s3/config.rs index 2e8a4979523592..d68fc051b73cb6 100644 --- a/src/sinks/aws_s3/config.rs +++ b/src/sinks/aws_s3/config.rs @@ -1,5 +1,3 @@ -use std::convert::TryInto; - use aws_sdk_s3::Client as S3Client; use tower::ServiceBuilder; use vector_lib::codecs::{ @@ -8,6 +6,7 @@ use vector_lib::codecs::{ }; use vector_lib::configurable::configurable_component; use vector_lib::sink::VectorSink; +use vector_lib::TimeZone; use super::sink::S3RequestOptions; use crate::{ @@ -23,8 +22,8 @@ use crate::{ sink::S3Sink, }, util::{ - BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression, ServiceBuilderExt, - TowerRequestConfig, + timezone_to_offset, BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression, + ServiceBuilderExt, TowerRequestConfig, }, Healthcheck, }, @@ -136,6 +135,10 @@ pub struct S3SinkConfig { skip_serializing_if = "crate::serde::skip_serializing_if_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub timezone: Option, } pub(super) fn default_key_prefix() -> String { @@ -163,6 +166,7 @@ impl GenerateConfig for S3SinkConfig { tls: Some(TlsConfig::default()), auth: AwsAuthentication::default(), acknowledgements: Default::default(), + timezone: Default::default(), }) .unwrap() } @@ -174,7 +178,7 @@ impl SinkConfig for S3SinkConfig { async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { let service = self.create_service(&cx.proxy).await?; let healthcheck = self.build_healthcheck(service.client())?; - let sink = self.build_processor(service)?; + let sink = self.build_processor(service, cx)?; Ok((sink, healthcheck)) } @@ -188,19 +192,30 @@ impl SinkConfig for S3SinkConfig { } impl S3SinkConfig { - pub fn build_processor(&self, service: S3Service) -> crate::Result { + pub fn build_processor( + &self, + service: S3Service, + cx: SinkContext, + ) -> crate::Result { // Build our S3 client/service, which is what we'll ultimately feed // requests into in order to ship files to S3. We build this here in // order to configure the client/service with retries, concurrency // limits, rate limits, and whatever else the client should have. - let request_limits = self.request.unwrap_with(&Default::default()); + let request_limits = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_limits, S3RetryLogic) .service(service); + let offset = self + .timezone + .or(cx.globals.timezone) + .and_then(timezone_to_offset); + // Configure our partitioning/batching. let batch_settings = self.batch.into_batcher_settings()?; - let key_prefix = self.key_prefix.clone().try_into()?; + + let key_prefix = Template::try_from(self.key_prefix.clone())?.with_tz_offset(offset); + let ssekms_key_id = self .options .ssekms_key_id @@ -208,6 +223,7 @@ impl S3SinkConfig { .cloned() .map(|ssekms_key_id| Template::try_from(ssekms_key_id.as_str())) .transpose()?; + let partitioner = S3KeyPartitioner::new(key_prefix, ssekms_key_id); let transformer = self.encoding.transformer(); @@ -222,6 +238,7 @@ impl S3SinkConfig { filename_append_uuid: self.filename_append_uuid, encoder: (transformer, encoder), compression: self.compression, + filename_tz_offset: offset, }; let sink = S3Sink::new(service, request_options, partitioner, batch_settings); diff --git a/src/sinks/aws_s3/integration_tests.rs b/src/sinks/aws_s3/integration_tests.rs index 27c53c1214429c..01c78dae085231 100644 --- a/src/sinks/aws_s3/integration_tests.rs +++ b/src/sinks/aws_s3/integration_tests.rs @@ -61,7 +61,7 @@ async fn s3_insert_message_into_with_flat_key_prefix() { config.key_prefix = "test-prefix".to_string(); let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); - let sink = config.build_processor(service).unwrap(); + let sink = config.build_processor(service, cx).unwrap(); let (lines, events, receiver) = make_events_batch(100, 10); run_and_assert_sink_compliance(sink, events, &AWS_SINK_TAGS).await; @@ -95,7 +95,7 @@ async fn s3_insert_message_into_with_folder_key_prefix() { config.key_prefix = "test-prefix/".to_string(); let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); - let sink = config.build_processor(service).unwrap(); + let sink = config.build_processor(service, cx).unwrap(); let (lines, events, receiver) = make_events_batch(100, 10); run_and_assert_sink_compliance(sink, events, &AWS_SINK_TAGS).await; @@ -132,7 +132,7 @@ async fn s3_insert_message_into_with_ssekms_key_id() { config.options.ssekms_key_id = Some("alias/aws/s3".to_string()); let service = config.create_service(&cx.globals.proxy).await.unwrap(); - let sink = config.build_processor(service).unwrap(); + let sink = config.build_processor(service, cx).unwrap(); let (lines, events, receiver) = make_events_batch(100, 10); run_and_assert_sink_compliance(sink, events, &AWS_SINK_TAGS).await; @@ -170,7 +170,7 @@ async fn s3_rotate_files_after_the_buffer_size_is_reached() { }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); - let sink = config.build_processor(service).unwrap(); + let sink = config.build_processor(service, cx).unwrap(); let (lines, _events) = random_lines_with_stream(100, 30, None); @@ -229,7 +229,7 @@ async fn s3_gzip() { let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); - let sink = config.build_processor(service).unwrap(); + let sink = config.build_processor(service, cx).unwrap(); let (lines, events, receiver) = make_events_batch(100, batch_size * batch_multiplier); run_and_assert_sink_compliance(sink, events, &AWS_SINK_TAGS).await; @@ -274,7 +274,7 @@ async fn s3_zstd() { let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); - let sink = config.build_processor(service).unwrap(); + let sink = config.build_processor(service, cx).unwrap(); let (lines, events, receiver) = make_events_batch(100, batch_size * batch_multiplier); run_and_assert_sink_compliance(sink, events, &AWS_SINK_TAGS).await; @@ -336,7 +336,7 @@ async fn s3_insert_message_into_object_lock() { let config = config(&bucket, 1000000); let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); - let sink = config.build_processor(service).unwrap(); + let sink = config.build_processor(service, cx).unwrap(); let (lines, events, receiver) = make_events_batch(100, 10); run_and_assert_sink_compliance(sink, events, &AWS_SINK_TAGS).await; @@ -368,7 +368,7 @@ async fn acknowledges_failures() { config.bucket = format!("BREAK{}IT", config.bucket); let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); - let sink = config.build_processor(service).unwrap(); + let sink = config.build_processor(service, cx).unwrap(); let (_lines, events, receiver) = make_events_batch(1, 1); run_and_assert_sink_error(sink, events, &COMPONENT_ERROR_TAGS).await; @@ -434,11 +434,12 @@ async fn s3_flush_on_exhaustion() { tls: Default::default(), auth: Default::default(), acknowledgements: Default::default(), + timezone: Default::default(), } }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); - let sink = config.build_processor(service).unwrap(); + let sink = config.build_processor(service, cx).unwrap(); let (lines, _events) = random_lines_with_stream(100, 2, None); // only generate two events (less than batch size) @@ -517,6 +518,7 @@ fn config(bucket: &str, batch_size: usize) -> S3SinkConfig { tls: Default::default(), auth: Default::default(), acknowledgements: Default::default(), + timezone: Default::default(), } } diff --git a/src/sinks/aws_s3/mod.rs b/src/sinks/aws_s3/mod.rs index fd3a3418b47166..3027de1bb62872 100644 --- a/src/sinks/aws_s3/mod.rs +++ b/src/sinks/aws_s3/mod.rs @@ -3,4 +3,4 @@ mod sink; mod integration_tests; -pub use self::config::S3SinkConfig; +pub use config::S3SinkConfig; diff --git a/src/sinks/aws_s3/sink.rs b/src/sinks/aws_s3/sink.rs index 553908064b680e..a4fc542437106a 100644 --- a/src/sinks/aws_s3/sink.rs +++ b/src/sinks/aws_s3/sink.rs @@ -1,7 +1,7 @@ use std::io; use bytes::Bytes; -use chrono::Utc; +use chrono::{FixedOffset, Utc}; use uuid::Uuid; use vector_lib::codecs::encoding::Framer; use vector_lib::event::Finalizable; @@ -32,6 +32,7 @@ pub struct S3RequestOptions { pub api_options: S3Options, pub encoder: (Transformer, Encoder), pub compression: Compression, + pub filename_tz_offset: Option, } impl RequestBuilder<(S3PartitionKey, Vec)> for S3RequestOptions { @@ -76,7 +77,14 @@ impl RequestBuilder<(S3PartitionKey, Vec)> for S3RequestOptions { payload: EncodeResult, ) -> Self::Request { let filename = { - let formatted_ts = Utc::now().format(self.filename_time_format.as_str()); + let formatted_ts = match self.filename_tz_offset { + Some(offset) => Utc::now() + .with_timezone(&offset) + .format(self.filename_time_format.as_str()), + None => Utc::now() + .with_timezone(&chrono::Utc) + .format(self.filename_time_format.as_str()), + }; self.filename_append_uuid .then(|| format!("{}-{}", formatted_ts, Uuid::new_v4().hyphenated())) diff --git a/src/sinks/aws_s_s/sink.rs b/src/sinks/aws_s_s/sink.rs index 41eb42f82e9f60..d933dcd7529450 100644 --- a/src/sinks/aws_s_s/sink.rs +++ b/src/sinks/aws_s_s/sink.rs @@ -40,9 +40,7 @@ where } async fn run_inner(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { - let request = self - .request - .unwrap_with(&TowerRequestConfig::default().timeout_secs(30)); + let request = self.request.into_settings(); let retry_logic: SSRetryLogic = super::retry::SSRetryLogic::new(); let service = tower::ServiceBuilder::new() .settings(request, retry_logic) diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index 859af08823b4b7..d429f88c1ad425 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -7,6 +7,7 @@ use vector_lib::configurable::configurable_component; use vector_lib::sensitive_string::SensitiveString; use super::request_builder::AzureBlobRequestOptions; +use crate::sinks::util::service::TowerRequestConfigDefaults; use crate::{ codecs::{Encoder, EncodingConfigWithFraming, SinkType}, config::{AcknowledgementsConfig, DataType, GenerateConfig, Input, SinkConfig, SinkContext}, @@ -24,6 +25,13 @@ use crate::{ Result, }; +#[derive(Clone, Copy, Debug)] +pub struct AzureBlobTowerRequestConfigDefaults; + +impl TowerRequestConfigDefaults for AzureBlobTowerRequestConfigDefaults { + const RATE_LIMIT_NUM: u64 = 250; +} + /// Configuration for the `azure_blob` sink. #[configurable_component(sink( "azure_blob", @@ -131,7 +139,7 @@ pub struct AzureBlobSinkConfig { #[configurable(derived)] #[serde(default)] - pub request: TowerRequestConfig, + pub request: TowerRequestConfig, #[configurable(derived)] #[serde( @@ -202,9 +210,7 @@ const DEFAULT_FILENAME_APPEND_UUID: bool = true; impl AzureBlobSinkConfig { pub fn build_processor(&self, client: Arc) -> crate::Result { - let request_limits = self - .request - .unwrap_with(&TowerRequestConfig::default().rate_limit_num(250)); + let request_limits = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_limits, AzureBlobRetryLogic) .service(AzureBlobService::new(client)); diff --git a/src/sinks/azure_monitor_logs/config.rs b/src/sinks/azure_monitor_logs/config.rs index e772fe53f762bf..258c6eacab2676 100644 --- a/src/sinks/azure_monitor_logs/config.rs +++ b/src/sinks/azure_monitor_logs/config.rs @@ -185,7 +185,7 @@ impl AzureMonitorLogsConfig { let retry_logic = HttpStatusRetryLogic::new(|res: &AzureMonitorLogsResponse| res.http_status); - let request_settings = self.request.unwrap_with(&Default::default()); + let request_settings = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) .service(service); diff --git a/src/sinks/clickhouse/config.rs b/src/sinks/clickhouse/config.rs index 0c8cc164633f8b..312c1b26467f70 100644 --- a/src/sinks/clickhouse/config.rs +++ b/src/sinks/clickhouse/config.rs @@ -95,7 +95,7 @@ impl SinkConfig for ClickhouseConfig { self.date_time_best_effort, ); - let request_limits = self.request.unwrap_with(&Default::default()); + let request_limits = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_limits, ClickhouseRetryLogic::default()) .service(service); diff --git a/src/sinks/clickhouse/integration_tests.rs b/src/sinks/clickhouse/integration_tests.rs index 05ee0ee4e333dd..79a6bc9c1985fd 100644 --- a/src/sinks/clickhouse/integration_tests.rs +++ b/src/sinks/clickhouse/integration_tests.rs @@ -50,7 +50,7 @@ async fn insert_events() { compression: Compression::None, batch, request: TowerRequestConfig { - retry_attempts: Some(1), + retry_attempts: 1, ..Default::default() }, ..Default::default() @@ -100,7 +100,7 @@ async fn skip_unknown_fields() { compression: Compression::None, batch, request: TowerRequestConfig { - retry_attempts: Some(1), + retry_attempts: 1, ..Default::default() }, ..Default::default() @@ -146,7 +146,7 @@ async fn insert_events_unix_timestamps() { encoding: Transformer::new(None, None, Some(TimestampFormat::Unix)).unwrap(), batch, request: TowerRequestConfig { - retry_attempts: Some(1), + retry_attempts: 1, ..Default::default() }, ..Default::default() diff --git a/src/sinks/databend/config.rs b/src/sinks/databend/config.rs index 4a29b97f4a8075..d9d4fbabfd5bde 100644 --- a/src/sinks/databend/config.rs +++ b/src/sinks/databend/config.rs @@ -116,7 +116,7 @@ impl SinkConfig for DatabendConfig { DatabendAPIClient::new(self.build_client(&cx)?, endpoint.clone(), auth.clone()); let healthcheck = select_one(health_client).boxed(); - let request_settings = self.request.unwrap_with(&TowerRequestConfig::default()); + let request_settings = self.request.into_settings(); let batch_settings = self.batch.into_batcher_settings()?; let database = config.database; diff --git a/src/sinks/datadog/events/config.rs b/src/sinks/datadog/events/config.rs index 166881667db91e..74f01c81ddbca3 100644 --- a/src/sinks/datadog/events/config.rs +++ b/src/sinks/datadog/events/config.rs @@ -72,7 +72,7 @@ impl DatadogEventsConfig { ); let request_opts = self.request; - let request_settings = request_opts.unwrap_with(&TowerRequestConfig::default()); + let request_settings = request_opts.into_settings(); let retry_logic = HttpStatusRetryLogic::new(|req: &DatadogEventsResponse| req.http_status); let service = ServiceBuilder::new() diff --git a/src/sinks/datadog/logs/config.rs b/src/sinks/datadog/logs/config.rs index e69181c38e65e8..3ed6a7938fed93 100644 --- a/src/sinks/datadog/logs/config.rs +++ b/src/sinks/datadog/logs/config.rs @@ -104,7 +104,7 @@ impl DatadogLogsConfig { dd_evp_origin: String, ) -> crate::Result { let default_api_key: Arc = Arc::from(self.dd_common.default_api_key.inner()); - let request_limits = self.request.tower.unwrap_with(&Default::default()); + let request_limits = self.request.tower.into_settings(); // We forcefully cap the provided batch configuration to the size/log line limits imposed by // the Datadog Logs API, but we still allow them to be lowered if need be. diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index b774beccc2001c..c0817a456a6759 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -31,9 +31,6 @@ use crate::{ pub const MAXIMUM_PAYLOAD_COMPRESSED_SIZE: usize = 3_200_000; pub const MAXIMUM_PAYLOAD_SIZE: usize = 62_914_560; -// TODO: revisit our concurrency and batching defaults -const DEFAULT_REQUEST_RETRY_ATTEMPTS: usize = 5; - #[derive(Clone, Copy, Debug, Default)] pub struct DatadogMetricsDefaultBatchSettings; @@ -65,12 +62,12 @@ impl SeriesApiVersion { } fn get_api_version_backwards_compatible() -> Self { static API_VERSION: OnceLock = OnceLock::new(); - *API_VERSION.get_or_init( - || match option_env!("VECTOR_TEMP_USE_DD_METRICS_SERIES_V1_API") { - Some(_) => Self::V1, - None => Self::V2, - }, - ) + *API_VERSION.get_or_init(|| { + match std::env::var("VECTOR_TEMP_USE_DD_METRICS_SERIES_V1_API") { + Ok(_) => Self::V1, + Err(_) => Self::V2, + } + }) } } @@ -232,9 +229,7 @@ impl DatadogMetricsConfig { let batcher_settings = self.batch.into_batcher_settings()?; // TODO: revisit our concurrency and batching defaults - let request_limits = self.request.unwrap_with( - &TowerRequestConfig::default().retry_attempts(DEFAULT_REQUEST_RETRY_ATTEMPTS), - ); + let request_limits = self.request.into_settings(); let endpoint_configuration = self.generate_metrics_endpoint_configuration()?; let service = ServiceBuilder::new() diff --git a/src/sinks/datadog/traces/config.rs b/src/sinks/datadog/traces/config.rs index b3fe500ba2311e..c28281b6ee3b3b 100644 --- a/src/sinks/datadog/traces/config.rs +++ b/src/sinks/datadog/traces/config.rs @@ -41,9 +41,6 @@ pub const BATCH_DEFAULT_TIMEOUT_SECS: f64 = 10.0; pub const PAYLOAD_LIMIT: usize = 3_200_000; -const DEFAULT_REQUEST_RETRY_ATTEMPTS: usize = 5; -const DEFAULT_REQUEST_RETRY_MAX_DURATION_SECS: u64 = 300; - #[derive(Clone, Copy, Debug, Default)] pub struct DatadogTracesDefaultBatchSettings; @@ -130,11 +127,7 @@ impl DatadogTracesConfig { pub fn build_sink(&self, client: HttpClient) -> crate::Result { let default_api_key: Arc = Arc::from(self.dd_common.default_api_key.inner()); - let request_limits = self.request.unwrap_with( - &TowerRequestConfig::default() - .retry_attempts(DEFAULT_REQUEST_RETRY_ATTEMPTS) - .retry_max_duration_secs(DEFAULT_REQUEST_RETRY_MAX_DURATION_SECS), - ); + let request_limits = self.request.into_settings(); let endpoints = self.generate_traces_endpoint_configuration()?; let batcher_settings = self diff --git a/src/sinks/elasticsearch/common.rs b/src/sinks/elasticsearch/common.rs index 27b71e527dae12..04e01fd457ebf1 100644 --- a/src/sinks/elasticsearch/common.rs +++ b/src/sinks/elasticsearch/common.rs @@ -19,7 +19,7 @@ use crate::{ ElasticsearchAuthConfig, ElasticsearchCommonMode, ElasticsearchConfig, ParseError, }, util::auth::Auth, - util::{http::RequestConfig, TowerRequestConfig, UriSerde}, + util::{http::RequestConfig, UriSerde}, HealthcheckError, }, tls::TlsSettings, @@ -89,10 +89,7 @@ impl ElasticsearchCommon { let mode = config.common_mode()?; - let tower_request = config - .request - .tower - .unwrap_with(&TowerRequestConfig::default()); + let tower_request = config.request.tower.into_settings(); let mut query_params = config.query.clone().unwrap_or_default(); query_params.insert( diff --git a/src/sinks/elasticsearch/config.rs b/src/sinks/elasticsearch/config.rs index 89b4095d52aafa..2a08bf42136f5c 100644 --- a/src/sinks/elasticsearch/config.rs +++ b/src/sinks/elasticsearch/config.rs @@ -23,7 +23,7 @@ use crate::{ }, util::{ http::RequestConfig, service::HealthConfig, BatchConfig, Compression, - RealtimeSizeBasedDefaultBatchSettings, TowerRequestConfig, + RealtimeSizeBasedDefaultBatchSettings, }, Healthcheck, VectorSink, }, @@ -482,10 +482,7 @@ impl SinkConfig for ElasticsearchConfig { let client = HttpClient::new(common.tls_settings.clone(), cx.proxy())?; - let request_limits = self - .request - .tower - .unwrap_with(&TowerRequestConfig::default()); + let request_limits = self.request.tower.into_settings(); let health_config = self.endpoint_health.clone().unwrap_or_default(); diff --git a/src/sinks/file/mod.rs b/src/sinks/file/mod.rs index 05609edc8e25a4..f7d94243ba35f5 100644 --- a/src/sinks/file/mod.rs +++ b/src/sinks/file/mod.rs @@ -22,7 +22,7 @@ use vector_lib::codecs::{ use vector_lib::configurable::configurable_component; use vector_lib::{ internal_event::{CountByteSize, EventsSent, InternalEventHandle as _, Output, Registered}, - EstimatedJsonEncodedSizeOf, + EstimatedJsonEncodedSizeOf, TimeZone, }; use crate::{ @@ -33,9 +33,10 @@ use crate::{ internal_events::{ FileBytesSent, FileInternalMetricsConfig, FileIoError, FileOpen, TemplateRenderingError, }, - sinks::util::StreamSink, + sinks::util::{timezone_to_offset, StreamSink}, template::Template, }; + mod bytes_path; use bytes_path::BytesPath; @@ -84,6 +85,10 @@ pub struct FileSinkConfig { )] pub acknowledgements: AcknowledgementsConfig, + #[configurable(derived)] + #[serde(default)] + pub timezone: Option, + #[configurable(derived)] #[serde(default)] internal_metrics: FileInternalMetricsConfig, @@ -97,6 +102,7 @@ impl GenerateConfig for FileSinkConfig { encoding: (None::, TextSerializerConfig::default()).into(), compression: Default::default(), acknowledgements: Default::default(), + timezone: Default::default(), internal_metrics: Default::default(), }) .unwrap() @@ -181,9 +187,9 @@ impl OutFile { impl SinkConfig for FileSinkConfig { async fn build( &self, - _cx: SinkContext, + cx: SinkContext, ) -> crate::Result<(super::VectorSink, super::Healthcheck)> { - let sink = FileSink::new(self)?; + let sink = FileSink::new(self, cx)?; Ok(( super::VectorSink::from_event_streamsink(sink), future::ok(()).boxed(), @@ -211,13 +217,18 @@ pub struct FileSink { } impl FileSink { - pub fn new(config: &FileSinkConfig) -> crate::Result { + pub fn new(config: &FileSinkConfig, cx: SinkContext) -> crate::Result { let transformer = config.encoding.transformer(); let (framer, serializer) = config.encoding.build(SinkType::StreamBased)?; let encoder = Encoder::::new(framer, serializer); + let offset = config + .timezone + .or(cx.globals.timezone) + .and_then(timezone_to_offset); + Ok(Self { - path: config.path.clone(), + path: config.path.clone().with_tz_offset(offset), transformer, encoder, idle_timeout: config.idle_timeout, @@ -464,7 +475,10 @@ mod tests { encoding: (None::, TextSerializerConfig::default()).into(), compression: Compression::None, acknowledgements: Default::default(), - internal_metrics: Default::default(), + timezone: Default::default(), + internal_metrics: FileInternalMetricsConfig { + include_file_tag: true, + }, }; let (input, _events) = random_lines_with_stream(100, 64, None); @@ -487,7 +501,10 @@ mod tests { encoding: (None::, TextSerializerConfig::default()).into(), compression: Compression::Gzip, acknowledgements: Default::default(), - internal_metrics: Default::default(), + timezone: Default::default(), + internal_metrics: FileInternalMetricsConfig { + include_file_tag: true, + }, }; let (input, _) = random_lines_with_stream(100, 64, None); @@ -510,7 +527,10 @@ mod tests { encoding: (None::, TextSerializerConfig::default()).into(), compression: Compression::Zstd, acknowledgements: Default::default(), - internal_metrics: Default::default(), + timezone: Default::default(), + internal_metrics: FileInternalMetricsConfig { + include_file_tag: true, + }, }; let (input, _) = random_lines_with_stream(100, 64, None); @@ -538,7 +558,10 @@ mod tests { encoding: (None::, TextSerializerConfig::default()).into(), compression: Compression::None, acknowledgements: Default::default(), - internal_metrics: Default::default(), + timezone: Default::default(), + internal_metrics: FileInternalMetricsConfig { + include_file_tag: true, + }, }; let (mut input, _events) = random_events_with_stream(32, 8, None); @@ -617,7 +640,10 @@ mod tests { encoding: (None::, TextSerializerConfig::default()).into(), compression: Compression::None, acknowledgements: Default::default(), - internal_metrics: Default::default(), + timezone: Default::default(), + internal_metrics: FileInternalMetricsConfig { + include_file_tag: true, + }, }; let (mut input, _events) = random_lines_with_stream(10, 64, None); @@ -626,7 +652,7 @@ mod tests { let sink_handle = tokio::spawn(async move { assert_sink_compliance(&FILE_SINK_TAGS, async move { - let sink = FileSink::new(&config).unwrap(); + let sink = FileSink::new(&config, SinkContext::default()).unwrap(); VectorSink::from_event_streamsink(sink) .run(Box::pin(rx.map(Into::into))) .await @@ -670,7 +696,7 @@ mod tests { async fn run_assert_sink(config: FileSinkConfig, events: impl Iterator + Send) { assert_sink_compliance(&FILE_SINK_TAGS, async move { - let sink = FileSink::new(&config).unwrap(); + let sink = FileSink::new(&config, SinkContext::default()).unwrap(); VectorSink::from_event_streamsink(sink) .run(Box::pin(stream::iter(events.map(Into::into)))) .await diff --git a/src/sinks/gcp/chronicle_unstructured.rs b/src/sinks/gcp/chronicle_unstructured.rs index a0fe97cd9073b1..bebc96b0d9a43f 100644 --- a/src/sinks/gcp/chronicle_unstructured.rs +++ b/src/sinks/gcp/chronicle_unstructured.rs @@ -22,6 +22,7 @@ use vector_lib::{ }; use vrl::value::Kind; +use crate::sinks::util::service::TowerRequestConfigDefaults; use crate::{ codecs::{self, EncodingConfig}, config::{GenerateConfig, SinkConfig, SinkContext}, @@ -97,6 +98,12 @@ impl SinkBatchSettings for ChronicleUnstructuredDefaultBatchSettings { const TIMEOUT_SECS: f64 = 15.0; } +#[derive(Clone, Copy, Debug)] +pub struct ChronicleUnstructuredTowerRequestConfigDefaults; + +impl TowerRequestConfigDefaults for ChronicleUnstructuredTowerRequestConfigDefaults { + const RATE_LIMIT_NUM: u64 = 1_000; +} /// Configuration for the `gcp_chronicle_unstructured` sink. #[configurable_component(sink( "gcp_chronicle_unstructured", @@ -132,7 +139,7 @@ pub struct ChronicleUnstructuredConfig { #[configurable(derived)] #[serde(default)] - pub request: TowerRequestConfig, + pub request: TowerRequestConfig, #[configurable(derived)] pub tls: Option, @@ -235,10 +242,7 @@ impl ChronicleUnstructuredConfig { ) -> crate::Result { use crate::sinks::util::service::ServiceBuilderExt; - let request = self.request.unwrap_with(&TowerRequestConfig { - rate_limit_num: Some(1000), - ..Default::default() - }); + let request = self.request.into_settings(); let batch_settings = self.batch.into_batcher_settings()?; diff --git a/src/sinks/gcp/cloud_storage.rs b/src/sinks/gcp/cloud_storage.rs index af1f0cd2ea41cd..e84519c57141f9 100644 --- a/src/sinks/gcp/cloud_storage.rs +++ b/src/sinks/gcp/cloud_storage.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, convert::TryFrom, io}; use bytes::Bytes; -use chrono::Utc; +use chrono::{FixedOffset, Utc}; use http::header::{HeaderName, HeaderValue}; use http::Uri; use indoc::indoc; @@ -12,9 +12,10 @@ use uuid::Uuid; use vector_lib::codecs::encoding::Framer; use vector_lib::configurable::configurable_component; use vector_lib::event::{EventFinalizers, Finalizable}; -use vector_lib::request_metadata::RequestMetadata; +use vector_lib::{request_metadata::RequestMetadata, TimeZone}; use crate::sinks::util::metadata::RequestMetadataBuilder; +use crate::sinks::util::service::TowerRequestConfigDefaults; use crate::{ codecs::{Encoder, EncodingConfigWithFraming, SinkType, Transformer}, config::{AcknowledgementsConfig, DataType, GenerateConfig, Input, SinkConfig, SinkContext}, @@ -32,8 +33,8 @@ use crate::{ }, util::{ batch::BatchConfig, partitioner::KeyPartitioner, request_builder::EncodeResult, - BulkSizeBasedDefaultBatchSettings, Compression, RequestBuilder, ServiceBuilderExt, - TowerRequestConfig, + timezone_to_offset, BulkSizeBasedDefaultBatchSettings, Compression, RequestBuilder, + ServiceBuilderExt, TowerRequestConfig, }, Healthcheck, VectorSink, }, @@ -48,6 +49,13 @@ pub enum GcsHealthcheckError { KeyPrefixTemplate { source: TemplateParseError }, } +#[derive(Clone, Copy, Debug)] +pub struct GcsTowerRequestConfigDefaults; + +impl TowerRequestConfigDefaults for GcsTowerRequestConfigDefaults { + const RATE_LIMIT_NUM: u64 = 1_000; +} + /// Configuration for the `gcp_cloud_storage` sink. #[configurable_component(sink( "gcp_cloud_storage", @@ -149,7 +157,7 @@ pub struct GcsSinkConfig { #[configurable(derived)] #[serde(default)] - request: TowerRequestConfig, + request: TowerRequestConfig, #[serde(flatten)] auth: GcpAuthConfig, @@ -164,6 +172,10 @@ pub struct GcsSinkConfig { skip_serializing_if = "crate::serde::skip_serializing_if_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub timezone: Option, } fn default_time_format() -> String { @@ -188,6 +200,7 @@ fn default_config(encoding: EncodingConfigWithFraming) -> GcsSinkConfig { auth: Default::default(), tls: Default::default(), acknowledgements: Default::default(), + timezone: Default::default(), } } @@ -218,7 +231,7 @@ impl SinkConfig for GcsSinkConfig { auth.clone(), )?; auth.spawn_regenerate_token(); - let sink = self.build_sink(client, base_url, auth)?; + let sink = self.build_sink(client, base_url, auth, cx)?; Ok((sink, healthcheck)) } @@ -238,11 +251,9 @@ impl GcsSinkConfig { client: HttpClient, base_url: String, auth: GcpAuthenticator, + cx: SinkContext, ) -> crate::Result { - let request = self.request.unwrap_with(&TowerRequestConfig { - rate_limit_num: Some(1000), - ..Default::default() - }); + let request = self.request.into_settings(); let batch_settings = self.batch.into_batcher_settings()?; @@ -254,7 +265,7 @@ impl GcsSinkConfig { .settings(request, GcsRetryLogic) .service(GcsService::new(client, base_url, auth)); - let request_settings = RequestSettings::new(self)?; + let request_settings = RequestSettings::new(self, cx)?; let sink = GcsSink::new(svc, request_settings, partitioner, batch_settings, protocol); @@ -284,6 +295,7 @@ struct RequestSettings { append_uuid: bool, encoder: (Transformer, Encoder), compression: Compression, + tz_offset: Option, } impl RequestBuilder<(String, Vec)> for RequestSettings { @@ -322,7 +334,12 @@ impl RequestBuilder<(String, Vec)> for RequestSettings { let (key, finalizers) = gcp_metadata; // TODO: pull the seconds from the last event let filename = { - let seconds = Utc::now().format(&self.time_format); + let seconds = match self.tz_offset { + Some(offset) => Utc::now().with_timezone(&offset).format(&self.time_format), + None => Utc::now() + .with_timezone(&chrono::Utc) + .format(&self.time_format), + }; if self.append_uuid { let uuid = Uuid::new_v4(); @@ -352,7 +369,7 @@ impl RequestBuilder<(String, Vec)> for RequestSettings { } impl RequestSettings { - fn new(config: &GcsSinkConfig) -> crate::Result { + fn new(config: &GcsSinkConfig, cx: SinkContext) -> crate::Result { let transformer = config.encoding.transformer(); let (framer, serializer) = config.encoding.build(SinkType::MessageBased)?; let encoder = Encoder::::new(framer, serializer); @@ -382,6 +399,11 @@ impl RequestSettings { .unwrap_or_else(|| config.compression.extension().into()); let time_format = config.filename_time_format.clone(); let append_uuid = config.filename_append_uuid; + let offset = config + .timezone + .or(cx.globals.timezone) + .and_then(timezone_to_offset); + Ok(Self { acl, content_type, @@ -393,6 +415,7 @@ impl RequestSettings { append_uuid, compression: config.compression, encoder: (transformer, encoder), + tz_offset: offset, }) } } @@ -442,7 +465,12 @@ mod tests { let config = default_config((None::, JsonSerializerConfig::default()).into()); let sink = config - .build_sink(client, mock_endpoint.to_string(), GcpAuthenticator::None) + .build_sink( + client, + mock_endpoint.to_string(), + GcpAuthenticator::None, + context, + ) .expect("failed to build sink"); let event = Event::Log(LogEvent::from("simple message")); @@ -470,11 +498,12 @@ mod tests { assert_eq!(key, "key: value"); } - fn request_settings(sink_config: &GcsSinkConfig) -> RequestSettings { - RequestSettings::new(sink_config).expect("Could not create request settings") + fn request_settings(sink_config: &GcsSinkConfig, context: SinkContext) -> RequestSettings { + RequestSettings::new(sink_config, context).expect("Could not create request settings") } fn build_request(extension: Option<&str>, uuid: bool, compression: Compression) -> GcsRequest { + let context = SinkContext::default(); let sink_config = GcsSinkConfig { key_prefix: Some("key/".into()), filename_time_format: "date".into(), @@ -499,7 +528,7 @@ mod tests { let mut byte_size = GroupedCountByteSize::new_untagged(); byte_size.add_event(&log, log.estimated_json_encoded_size_of()); - let request_settings = request_settings(&sink_config); + let request_settings = request_settings(&sink_config, context); let (metadata, metadata_request_builder, _events) = request_settings.split_input((key, vec![log])); let payload = EncodeResult::uncompressed(Bytes::new(), byte_size); diff --git a/src/sinks/gcp/pubsub.rs b/src/sinks/gcp/pubsub.rs index 658b62061de632..876fe9ba8e5acc 100644 --- a/src/sinks/gcp/pubsub.rs +++ b/src/sinks/gcp/pubsub.rs @@ -123,7 +123,7 @@ impl SinkConfig for PubsubConfig { .validate()? .limit_max_bytes(MAX_BATCH_PAYLOAD_SIZE)? .into_batch_settings()?; - let request_settings = self.request.unwrap_with(&Default::default()); + let request_settings = self.request.into_settings(); let tls_settings = TlsSettings::from_options(&self.tls)?; let client = HttpClient::new(tls_settings, cx.proxy())?; diff --git a/src/sinks/gcp/stackdriver/logs/config.rs b/src/sinks/gcp/stackdriver/logs/config.rs index 9e3a82eb4c3b57..164570ecb04270 100644 --- a/src/sinks/gcp/stackdriver/logs/config.rs +++ b/src/sinks/gcp/stackdriver/logs/config.rs @@ -9,6 +9,7 @@ use crate::{ prelude::*, util::{ http::{http_response_retry_logic, HttpService}, + service::TowerRequestConfigDefaults, BoxedRawValue, RealtimeSizeBasedDefaultBatchSettings, }, }, @@ -31,6 +32,13 @@ enum HealthcheckError { NotFound, } +#[derive(Clone, Copy, Debug)] +pub struct StackdriverTowerRequestConfigDefaults; + +impl TowerRequestConfigDefaults for StackdriverTowerRequestConfigDefaults { + const RATE_LIMIT_NUM: u64 = 1_000; +} + /// Configuration for the `gcp_stackdriver_logs` sink. #[configurable_component(sink( "gcp_stackdriver_logs", @@ -85,7 +93,7 @@ pub(super) struct StackdriverConfig { #[configurable(derived)] #[serde(default)] - pub(super) request: TowerRequestConfig, + pub(super) request: TowerRequestConfig, #[configurable(derived)] pub(super) tls: Option, @@ -214,11 +222,7 @@ impl SinkConfig for StackdriverConfig { .limit_max_bytes(MAX_BATCH_PAYLOAD_SIZE)? .into_batcher_settings()?; - let request_limits = self.request.unwrap_with( - &TowerRequestConfig::default() - .rate_limit_duration_secs(1) - .rate_limit_num(1000), - ); + let request_limits = self.request.into_settings(); let tls_settings = TlsSettings::from_options(&self.tls)?; let client = HttpClient::new(tls_settings, cx.proxy())?; diff --git a/src/sinks/gcp/stackdriver/metrics/config.rs b/src/sinks/gcp/stackdriver/metrics/config.rs index 99d03d114fd293..fa0f86d8ce9f67 100644 --- a/src/sinks/gcp/stackdriver/metrics/config.rs +++ b/src/sinks/gcp/stackdriver/metrics/config.rs @@ -8,7 +8,10 @@ use crate::{ sinks::{ gcp, prelude::*, - util::http::{http_response_retry_logic, HttpService, HttpServiceRequestBuilder}, + util::{ + http::{http_response_retry_logic, HttpService, HttpServiceRequestBuilder}, + service::TowerRequestConfigDefaults, + }, }, }; @@ -17,6 +20,13 @@ use super::{ sink::StackdriverMetricsSink, }; +#[derive(Clone, Copy, Debug)] +pub struct StackdriverMetricsTowerRequestConfigDefaults; + +impl TowerRequestConfigDefaults for StackdriverMetricsTowerRequestConfigDefaults { + const RATE_LIMIT_NUM: u64 = 1_000; +} + /// Configuration for the `gcp_stackdriver_metrics` sink. #[configurable_component(sink( "gcp_stackdriver_metrics", @@ -49,7 +59,7 @@ pub struct StackdriverConfig { #[configurable(derived)] #[serde(default)] - pub(super) request: TowerRequestConfig, + pub(super) request: TowerRequestConfig, #[configurable(derived)] #[serde(default)] @@ -98,11 +108,7 @@ impl SinkConfig for StackdriverConfig { }, }; - let request_limits = self.request.unwrap_with( - &TowerRequestConfig::default() - .rate_limit_duration_secs(1) - .rate_limit_num(1000), - ); + let request_limits = self.request.into_settings(); let uri: Uri = format!( "{}/v3/projects/{}/timeSeries", diff --git a/src/sinks/greptimedb/mod.rs b/src/sinks/greptimedb/mod.rs index ab1d132adc98bb..4891ea32b6cc84 100644 --- a/src/sinks/greptimedb/mod.rs +++ b/src/sinks/greptimedb/mod.rs @@ -116,7 +116,7 @@ impl_generate_config_from_default!(GreptimeDBConfig); #[async_trait::async_trait] impl SinkConfig for GreptimeDBConfig { async fn build(&self, _cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { - let request_settings = self.request.unwrap_with(&TowerRequestConfig::default()); + let request_settings = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_settings, GreptimeDBRetryLogic) .service(service::GreptimeDBService::try_new(self)?); diff --git a/src/sinks/honeycomb/config.rs b/src/sinks/honeycomb/config.rs index 6dda01074f7f03..50e656ae2b1976 100644 --- a/src/sinks/honeycomb/config.rs +++ b/src/sinks/honeycomb/config.rs @@ -113,7 +113,7 @@ impl SinkConfig for HoneycombConfig { let service = HttpService::new(client.clone(), honeycomb_service_request_builder); - let request_limits = self.request.unwrap_with(&TowerRequestConfig::default()); + let request_limits = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_limits, http_response_retry_logic()) diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index 0bc67044012a44..a9b7b2aca5a07a 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -286,7 +286,7 @@ impl SinkConfig for HttpSinkConfig { let service = HttpService::new(client, http_sink_request_builder); - let request_limits = self.request.tower.unwrap_with(&Default::default()); + let request_limits = self.request.tower.into_settings(); let service = ServiceBuilder::new() .settings(request_limits, http_response_retry_logic()) diff --git a/src/sinks/influxdb/logs.rs b/src/sinks/influxdb/logs.rs index bd5c8460407f3b..3901ba1d97166f 100644 --- a/src/sinks/influxdb/logs.rs +++ b/src/sinks/influxdb/logs.rs @@ -171,10 +171,7 @@ impl SinkConfig for InfluxDbLogsConfig { let healthcheck = self.healthcheck(client.clone())?; let batch = self.batch.into_batch_settings()?; - let request = self.request.unwrap_with(&TowerRequestConfig { - retry_attempts: Some(5), - ..Default::default() - }); + let request = self.request.into_settings(); let settings = influxdb_settings( self.influxdb1_settings.clone(), diff --git a/src/sinks/influxdb/metrics.rs b/src/sinks/influxdb/metrics.rs index 597c3e21906023..6b8232c70b5ef5 100644 --- a/src/sinks/influxdb/metrics.rs +++ b/src/sinks/influxdb/metrics.rs @@ -159,10 +159,7 @@ impl InfluxDbSvc { let protocol_version = settings.protocol_version(); let batch = config.batch.into_batch_settings()?; - let request = config.request.unwrap_with(&TowerRequestConfig { - retry_attempts: Some(5), - ..Default::default() - }); + let request = config.request.into_settings(); let uri = settings.write_uri(endpoint)?; diff --git a/src/sinks/loki/sink.rs b/src/sinks/loki/sink.rs index cc3127415bf62b..f244c1b5f025a8 100644 --- a/src/sinks/loki/sink.rs +++ b/src/sinks/loki/sink.rs @@ -389,9 +389,9 @@ impl LokiSink { // requires in-order processing for version >= 2.4, instead we just keep the static limit // of 1 for now. let request_limits = match config.out_of_order_action { - OutOfOrderAction::Accept => config.request.unwrap_with(&Default::default()), + OutOfOrderAction::Accept => config.request.into_settings(), OutOfOrderAction::Drop | OutOfOrderAction::RewriteTimestamp => { - let mut settings = config.request.unwrap_with(&Default::default()); + let mut settings = config.request.into_settings(); settings.concurrency = Some(1); settings } diff --git a/src/sinks/mezmo.rs b/src/sinks/mezmo.rs index 149b2c024c8129..b28e73f39c1b7d 100644 --- a/src/sinks/mezmo.rs +++ b/src/sinks/mezmo.rs @@ -163,7 +163,7 @@ impl SinkConfig for MezmoConfig { &self, cx: SinkContext, ) -> crate::Result<(super::VectorSink, super::Healthcheck)> { - let request_settings = self.request.unwrap_with(&TowerRequestConfig::default()); + let request_settings = self.request.into_settings(); let batch_settings = self.batch.into_batch_settings()?; let client = HttpClient::new(None, cx.proxy())?; diff --git a/src/sinks/nats/config.rs b/src/sinks/nats/config.rs index db78aff4a9bddc..608e4895d12fab 100644 --- a/src/sinks/nats/config.rs +++ b/src/sinks/nats/config.rs @@ -5,11 +5,18 @@ use vector_lib::tls::TlsEnableableConfig; use crate::{ nats::{from_tls_auth_config, NatsAuthConfig, NatsConfigError}, - sinks::prelude::*, + sinks::{prelude::*, util::service::TowerRequestConfigDefaults}, }; use super::{sink::NatsSink, ConfigSnafu, ConnectSnafu, NatsError}; +#[derive(Clone, Copy, Debug)] +pub struct NatsTowerRequestConfigDefaults; + +impl TowerRequestConfigDefaults for NatsTowerRequestConfigDefaults { + const CONCURRENCY: Concurrency = Concurrency::None; +} + /// Configuration for the `nats` sink. #[configurable_component(sink( "nats", @@ -68,7 +75,7 @@ pub struct NatsSinkConfig { #[configurable(derived)] #[serde(default)] - pub(super) request: TowerRequestConfig, + pub(super) request: TowerRequestConfig, } fn default_name() -> String { diff --git a/src/sinks/nats/sink.rs b/src/sinks/nats/sink.rs index f6bea42919e9db..f2f4524b6ecfb2 100644 --- a/src/sinks/nats/sink.rs +++ b/src/sinks/nats/sink.rs @@ -5,7 +5,7 @@ use snafu::ResultExt; use crate::sinks::prelude::*; use super::{ - config::NatsSinkConfig, + config::{NatsSinkConfig, NatsTowerRequestConfigDefaults}, request_builder::{NatsEncoder, NatsRequestBuilder}, service::{NatsResponse, NatsService}, EncodingSnafu, NatsError, @@ -17,7 +17,7 @@ pub(super) struct NatsEvent { } pub(super) struct NatsSink { - request: TowerRequestConfig, + request: TowerRequestConfig, transformer: Transformer, encoder: Encoder<()>, connection: Arc, @@ -59,10 +59,7 @@ impl NatsSink { } async fn run_inner(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { - let request = self.request.unwrap_with(&TowerRequestConfig { - concurrency: Concurrency::Fixed(1), - ..Default::default() - }); + let request = self.request.into_settings(); let request_builder = NatsRequestBuilder { encoder: NatsEncoder { diff --git a/src/sinks/new_relic/config.rs b/src/sinks/new_relic/config.rs index 6ac338929a0aa9..f13015add1da88 100644 --- a/src/sinks/new_relic/config.rs +++ b/src/sinks/new_relic/config.rs @@ -141,7 +141,7 @@ impl SinkConfig for NewRelicConfig { .limit_max_events(self.batch.max_events.unwrap_or(100))? .into_batcher_settings()?; - let request_limits = self.request.unwrap_with(&Default::default()); + let request_limits = self.request.into_settings(); let tls_settings = TlsSettings::from_options(&None)?; let client = HttpClient::new(tls_settings, &cx.proxy)?; let credentials = Arc::from(NewRelicCredentials::from(self)); diff --git a/src/sinks/prometheus/remote_write/config.rs b/src/sinks/prometheus/remote_write/config.rs index 8f9f065642bdfa..954103b6a6631e 100644 --- a/src/sinks/prometheus/remote_write/config.rs +++ b/src/sinks/prometheus/remote_write/config.rs @@ -135,7 +135,7 @@ impl SinkConfig for RemoteWriteConfig { async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { let endpoint = self.endpoint.parse::().context(UriParseSnafu)?; let tls_settings = TlsSettings::from_options(&self.tls)?; - let request_settings = self.request.unwrap_with(&TowerRequestConfig::default()); + let request_settings = self.request.into_settings(); let buckets = self.buckets.clone(); let quantiles = self.quantiles.clone(); let default_namespace = self.default_namespace.clone(); diff --git a/src/sinks/redis/config.rs b/src/sinks/redis/config.rs index 0504cda3ceb303..b478fc0caba5ad 100644 --- a/src/sinks/redis/config.rs +++ b/src/sinks/redis/config.rs @@ -1,10 +1,17 @@ use redis::{aio::ConnectionManager, RedisResult}; use snafu::prelude::*; -use crate::sinks::prelude::*; +use crate::sinks::{prelude::*, util::service::TowerRequestConfigDefaults}; use super::{sink::RedisSink, RedisCreateFailedSnafu}; +#[derive(Clone, Copy, Debug)] +pub struct RedisTowerRequestConfigDefaults; + +impl TowerRequestConfigDefaults for RedisTowerRequestConfigDefaults { + const CONCURRENCY: Concurrency = Concurrency::None; +} + /// Redis data type to store messages in. #[configurable_component] #[derive(Clone, Copy, Debug, Derivative)] @@ -98,7 +105,7 @@ pub struct RedisSinkConfig { #[configurable(derived)] #[serde(default)] - pub(super) request: TowerRequestConfig, + pub(super) request: TowerRequestConfig, #[configurable(derived)] #[serde( diff --git a/src/sinks/redis/integration_tests.rs b/src/sinks/redis/integration_tests.rs index 1461b3896a25fc..4cc1856cb62d36 100644 --- a/src/sinks/redis/integration_tests.rs +++ b/src/sinks/redis/integration_tests.rs @@ -44,7 +44,7 @@ async fn redis_sink_list_lpush() { }), batch: BatchConfig::default(), request: TowerRequestConfig { - rate_limit_num: Some(u64::MAX), + rate_limit_num: u64::MAX, ..Default::default() }, acknowledgements: Default::default(), @@ -108,7 +108,7 @@ async fn redis_sink_list_rpush() { }), batch: BatchConfig::default(), request: TowerRequestConfig { - rate_limit_num: Some(u64::MAX), + rate_limit_num: u64::MAX, ..Default::default() }, acknowledgements: Default::default(), @@ -186,7 +186,7 @@ async fn redis_sink_channel() { list_option: None, batch: BatchConfig::default(), request: TowerRequestConfig { - rate_limit_num: Some(u64::MAX), + rate_limit_num: u64::MAX, ..Default::default() }, acknowledgements: Default::default(), @@ -262,7 +262,7 @@ async fn redis_sink_channel_data_volume_tags() { list_option: None, batch: BatchConfig::default(), request: TowerRequestConfig { - rate_limit_num: Some(u64::MAX), + rate_limit_num: u64::MAX, ..Default::default() }, acknowledgements: Default::default(), diff --git a/src/sinks/redis/sink.rs b/src/sinks/redis/sink.rs index 128ce05adddef1..5cac95f7698d7c 100644 --- a/src/sinks/redis/sink.rs +++ b/src/sinks/redis/sink.rs @@ -2,20 +2,17 @@ use std::future; use redis::{aio::ConnectionManager, RedisError}; -use crate::sinks::{ - prelude::*, - util::{retries::RetryAction, Concurrency}, -}; +use crate::sinks::{prelude::*, util::retries::RetryAction}; use super::{ - config::{DataTypeConfig, RedisSinkConfig}, + config::{DataTypeConfig, RedisSinkConfig, RedisTowerRequestConfigDefaults}, request_builder::request_builder, service::{RedisResponse, RedisService}, RedisEvent, }; pub(super) struct RedisSink { - request: TowerRequestConfig, + request: TowerRequestConfig, encoder: crate::codecs::Encoder<()>, transformer: crate::codecs::Transformer, conn: ConnectionManager, @@ -70,10 +67,7 @@ impl RedisSink { } async fn run_inner(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { - let request = self.request.unwrap_with(&TowerRequestConfig { - concurrency: Concurrency::Fixed(1), - ..Default::default() - }); + let request = self.request.into_settings(); let service = RedisService { conn: self.conn.clone(), diff --git a/src/sinks/sematext/metrics.rs b/src/sinks/sematext/metrics.rs index 14a3f37470709e..33138fdfd99b6e 100644 --- a/src/sinks/sematext/metrics.rs +++ b/src/sinks/sematext/metrics.rs @@ -166,10 +166,7 @@ impl SematextMetricsService { client: HttpClient, ) -> Result { let batch = config.batch.into_batch_settings()?; - let request = config.request.unwrap_with(&TowerRequestConfig { - retry_attempts: Some(5), - ..Default::default() - }); + let request = config.request.into_settings(); let http_service = HttpBatchService::new(client, create_build_request(endpoint)); let sematext_service = SematextMetricsService { config, diff --git a/src/sinks/splunk_hec/logs/config.rs b/src/sinks/splunk_hec/logs/config.rs index 2bd2515f237c07..4fa130187f7c67 100644 --- a/src/sinks/splunk_hec/logs/config.rs +++ b/src/sinks/splunk_hec/logs/config.rs @@ -239,7 +239,7 @@ impl HecLogsSinkConfig { compression: self.compression, }; - let request_settings = self.request.unwrap_with(&TowerRequestConfig::default()); + let request_settings = self.request.into_settings(); let http_request_builder = Arc::new(HttpRequestBuilder::new( self.endpoint.clone(), self.endpoint_target, diff --git a/src/sinks/splunk_hec/metrics/config.rs b/src/sinks/splunk_hec/metrics/config.rs index cc64656c68affa..a100830585a218 100644 --- a/src/sinks/splunk_hec/metrics/config.rs +++ b/src/sinks/splunk_hec/metrics/config.rs @@ -181,7 +181,7 @@ impl HecMetricsSinkConfig { compression: self.compression, }; - let request_settings = self.request.unwrap_with(&TowerRequestConfig::default()); + let request_settings = self.request.into_settings(); let http_request_builder = Arc::new(HttpRequestBuilder::new( self.endpoint.clone(), EndpointTarget::default(), diff --git a/src/sinks/util/adaptive_concurrency/controller.rs b/src/sinks/util/adaptive_concurrency/controller.rs index 8f14408e2bac67..7bc6cec55cc303 100644 --- a/src/sinks/util/adaptive_concurrency/controller.rs +++ b/src/sinks/util/adaptive_concurrency/controller.rs @@ -68,7 +68,7 @@ impl Controller { // If a `concurrency` is specified, it becomes both the // current limit and the maximum, effectively bypassing all the // mechanisms. Otherwise, the current limit is set to 1 and the - // maximum to MAX_CONCURRENCY. + // maximum to `settings.max_concurrency_limit`. let current_limit = concurrency.unwrap_or(settings.initial_concurrency); Self { semaphore: Arc::new(ShrinkableSemaphore::new(current_limit)), @@ -226,7 +226,7 @@ impl Controller { // concurrency limit. Note that we only check this if we had // requests to go beyond the current limit to prevent // increasing the limit beyond what we have evidence for. - if inner.current_limit < super::MAX_CONCURRENCY + if inner.current_limit < self.settings.max_concurrency_limit && inner.reached_limit && !inner.had_back_pressure && current_rtt.is_some() diff --git a/src/sinks/util/adaptive_concurrency/mod.rs b/src/sinks/util/adaptive_concurrency/mod.rs index dedbc8d1221b1e..6b1e21439ffaaa 100644 --- a/src/sinks/util/adaptive_concurrency/mod.rs +++ b/src/sinks/util/adaptive_concurrency/mod.rs @@ -9,10 +9,6 @@ mod service; #[cfg(test)] pub mod tests; -// Make sure to update the max range of the `AdaptiveConcurrencySettings::initial_concurrency` when changing -// this constant. -pub(super) const MAX_CONCURRENCY: usize = 200; - pub(crate) use layer::AdaptiveConcurrencyLimitLayer; pub(crate) use service::AdaptiveConcurrencyLimit; use vector_lib::configurable::configurable_component; @@ -36,7 +32,7 @@ pub struct AdaptiveConcurrencySettings { /// It is recommended to set this value to your service's average limit if you're seeing that it takes a /// long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the /// `adaptive_concurrency_limit` metric. - #[configurable(validation(range(min = 1, max = 200)))] + #[configurable(validation(range(min = 1)))] #[serde(default = "default_initial_concurrency")] pub(super) initial_concurrency: usize, @@ -72,6 +68,13 @@ pub struct AdaptiveConcurrencySettings { #[configurable(validation(range(min = 0.0)))] #[serde(default = "default_rtt_deviation_scale")] pub(super) rtt_deviation_scale: f64, + + /// The maximum concurrency limit. + /// + /// The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + #[configurable(validation(range(min = 1)))] + #[serde(default = "default_max_concurrency_limit")] + pub(super) max_concurrency_limit: usize, } const fn default_initial_concurrency() -> usize { @@ -90,10 +93,8 @@ const fn default_rtt_deviation_scale() -> f64 { 2.5 } -impl AdaptiveConcurrencySettings { - pub const fn max_concurrency() -> usize { - MAX_CONCURRENCY - } +const fn default_max_concurrency_limit() -> usize { + 200 } impl Default for AdaptiveConcurrencySettings { @@ -103,6 +104,7 @@ impl Default for AdaptiveConcurrencySettings { decrease_ratio: default_decrease_ratio(), ewma_alpha: default_ewma_alpha(), rtt_deviation_scale: default_rtt_deviation_scale(), + max_concurrency_limit: default_max_concurrency_limit(), } } } diff --git a/src/sinks/util/adaptive_concurrency/tests.rs b/src/sinks/util/adaptive_concurrency/tests.rs index 0aa059fd11a813..12d69fa103f646 100644 --- a/src/sinks/util/adaptive_concurrency/tests.rs +++ b/src/sinks/util/adaptive_concurrency/tests.rs @@ -35,8 +35,8 @@ use crate::{ metrics, sinks::{ util::{ - retries::RetryLogic, BatchSettings, Concurrency, EncodedEvent, EncodedLength, - TowerRequestConfig, VecBuffer, + retries::{JitterMode, RetryLogic}, + BatchSettings, Concurrency, EncodedEvent, EncodedLength, TowerRequestConfig, VecBuffer, }, Healthcheck, VectorSink, }, @@ -178,7 +178,7 @@ impl SinkConfig for TestConfig { batch_settings.size.events = 1; batch_settings.timeout = Duration::from_secs(9999); - let request = self.request.unwrap_with(&TowerRequestConfig::default()); + let request = self.request.into_settings(); let sink = request .batch_sink( TestRetryLogic, @@ -415,8 +415,9 @@ async fn run_test(params: TestParams) -> TestResults { let test_config = TestConfig { request: TowerRequestConfig { concurrency: params.concurrency, - rate_limit_num: Some(9999), - timeout_secs: Some(1), + rate_limit_num: 9999, + timeout_secs: 1, + retry_jitter_mode: JitterMode::None, ..Default::default() }, params, diff --git a/src/sinks/util/mod.rs b/src/sinks/util/mod.rs index f0a233b7037e92..23474327cdfc5a 100644 --- a/src/sinks/util/mod.rs +++ b/src/sinks/util/mod.rs @@ -52,9 +52,10 @@ pub use service::{ pub use sink::{BatchSink, PartitionBatchSink, StreamSink}; use snafu::Snafu; pub use uri::UriSerde; -use vector_lib::json_size::JsonSize; +use vector_lib::{json_size::JsonSize, TimeZone}; use crate::event::EventFinalizers; +use chrono::{FixedOffset, Offset, Utc}; #[derive(Debug, Snafu)] enum SinkBuildError { @@ -134,3 +135,10 @@ impl ElementCount for Vec { self.len() } } + +pub fn timezone_to_offset(tz: TimeZone) -> Option { + match tz { + TimeZone::Local => Some(*Utc::now().with_timezone(&chrono::Local).offset()), + TimeZone::Named(tz) => Some(Utc::now().with_timezone(&tz).offset().fix()), + } +} diff --git a/src/sinks/util/retries.rs b/src/sinks/util/retries.rs index 8bed68bfe61ce0..cccfbd744494f3 100644 --- a/src/sinks/util/retries.rs +++ b/src/sinks/util/retries.rs @@ -10,6 +10,7 @@ use std::{ use futures::FutureExt; use tokio::time::{sleep, Sleep}; use tower::{retry::Policy, timeout::error::Elapsed}; +use vector_lib::configurable::configurable_component; use crate::Error; @@ -34,50 +35,87 @@ pub trait RetryLogic: Clone + Send + Sync + 'static { } } +/// The jitter mode to use for retry backoff behavior. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default)] +pub enum JitterMode { + /// No jitter. + None, + + /// Full jitter. + /// + /// The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + /// strategy. + /// + /// Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + /// of creating accidental denial of service (DoS) conditions against your own systems when + /// many clients are recovering from a failure state. + #[default] + Full, +} + #[derive(Debug, Clone)] -pub struct FixedRetryPolicy { +pub struct FibonacciRetryPolicy { remaining_attempts: usize, previous_duration: Duration, current_duration: Duration, + jitter_mode: JitterMode, + current_jitter_duration: Duration, max_duration: Duration, logic: L, } pub struct RetryPolicyFuture { delay: Pin>, - policy: FixedRetryPolicy, + policy: FibonacciRetryPolicy, } -impl FixedRetryPolicy { - pub const fn new( +impl FibonacciRetryPolicy { + pub fn new( remaining_attempts: usize, initial_backoff: Duration, max_duration: Duration, logic: L, + jitter_mode: JitterMode, ) -> Self { - FixedRetryPolicy { + FibonacciRetryPolicy { remaining_attempts, previous_duration: Duration::from_secs(0), current_duration: initial_backoff, + jitter_mode, + current_jitter_duration: Self::add_full_jitter(initial_backoff), max_duration, logic, } } - fn advance(&self) -> FixedRetryPolicy { - let next_duration: Duration = self.previous_duration + self.current_duration; + fn add_full_jitter(d: Duration) -> Duration { + let jitter = (rand::random::() % (d.as_millis() as u64)) + 1; + Duration::from_millis(jitter) + } + + fn advance(&self) -> FibonacciRetryPolicy { + let next_duration: Duration = cmp::min( + self.previous_duration + self.current_duration, + self.max_duration, + ); - FixedRetryPolicy { + FibonacciRetryPolicy { remaining_attempts: self.remaining_attempts - 1, previous_duration: self.current_duration, - current_duration: cmp::min(next_duration, self.max_duration), + current_duration: next_duration, + current_jitter_duration: Self::add_full_jitter(next_duration), + jitter_mode: self.jitter_mode, max_duration: self.max_duration, logic: self.logic.clone(), } } const fn backoff(&self) -> Duration { - self.current_duration + match self.jitter_mode { + JitterMode::None => self.current_duration, + JitterMode::Full => self.current_jitter_duration, + } } fn build_retry(&self) -> RetryPolicyFuture { @@ -89,7 +127,7 @@ impl FixedRetryPolicy { } } -impl Policy for FixedRetryPolicy +impl Policy for FibonacciRetryPolicy where Req: Clone, L: RetryLogic, @@ -168,7 +206,7 @@ where impl Unpin for RetryPolicyFuture {} impl Future for RetryPolicyFuture { - type Output = FixedRetryPolicy; + type Output = FibonacciRetryPolicy; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { std::task::ready!(self.delay.poll_unpin(cx)); @@ -288,11 +326,12 @@ mod tests { time::pause(); - let policy = FixedRetryPolicy::new( + let policy = FibonacciRetryPolicy::new( 5, Duration::from_secs(1), Duration::from_secs(10), SvcRetryLogic, + JitterMode::None, ); let (mut svc, mut handle) = mock::spawn_layer(RetryLayer::new(policy)); @@ -317,11 +356,12 @@ mod tests { async fn service_error_no_retry() { trace_init(); - let policy = FixedRetryPolicy::new( + let policy = FibonacciRetryPolicy::new( 5, Duration::from_secs(1), Duration::from_secs(10), SvcRetryLogic, + JitterMode::None, ); let (mut svc, mut handle) = mock::spawn_layer(RetryLayer::new(policy)); @@ -339,11 +379,12 @@ mod tests { time::pause(); - let policy = FixedRetryPolicy::new( + let policy = FibonacciRetryPolicy::new( 5, Duration::from_secs(1), Duration::from_secs(10), SvcRetryLogic, + JitterMode::None, ); let (mut svc, mut handle) = mock::spawn_layer(RetryLayer::new(policy)); @@ -363,11 +404,12 @@ mod tests { #[test] fn backoff_grows_to_max() { - let mut policy = FixedRetryPolicy::new( + let mut policy = FibonacciRetryPolicy::new( 10, Duration::from_secs(1), Duration::from_secs(10), SvcRetryLogic, + JitterMode::None, ); assert_eq!(Duration::from_secs(1), policy.backoff()); @@ -393,6 +435,49 @@ mod tests { assert_eq!(Duration::from_secs(10), policy.backoff()); } + #[test] + fn backoff_grows_to_max_with_jitter() { + let max_duration = Duration::from_secs(10); + let mut policy = FibonacciRetryPolicy::new( + 10, + Duration::from_secs(1), + max_duration, + SvcRetryLogic, + JitterMode::Full, + ); + + let expected_fib = [1, 1, 2, 3, 5, 8]; + + for (i, &exp_fib_secs) in expected_fib.iter().enumerate() { + let backoff = policy.backoff(); + let upper_bound = Duration::from_secs(exp_fib_secs); + + // Check if the backoff is within the expected range, considering the jitter + assert!( + !backoff.is_zero() && backoff <= upper_bound, + "Attempt {}: Expected backoff to be within 0 and {:?}, got {:?}", + i + 1, + upper_bound, + backoff + ); + + policy = policy.advance(); + } + + // Once the max backoff is reached, it should not exceed the max backoff. + for _ in 0..4 { + let backoff = policy.backoff(); + assert!( + !backoff.is_zero() && backoff <= max_duration, + "Expected backoff to not exceed {:?}, got {:?}", + max_duration, + backoff + ); + + policy = policy.advance(); + } + } + #[derive(Debug, Clone)] struct SvcRetryLogic; diff --git a/src/sinks/util/service.rs b/src/sinks/util/service.rs index 7c70b970bf7367..17e9fc303bd9af 100644 --- a/src/sinks/util/service.rs +++ b/src/sinks/util/service.rs @@ -15,7 +15,7 @@ use tower::{ use vector_lib::configurable::configurable_component; pub use crate::sinks::util::service::{ - concurrency::{concurrency_is_adaptive, Concurrency}, + concurrency::Concurrency, health::{HealthConfig, HealthLogic, HealthService}, map::Map, }; @@ -25,7 +25,7 @@ use crate::{ adaptive_concurrency::{ AdaptiveConcurrencyLimit, AdaptiveConcurrencyLimitLayer, AdaptiveConcurrencySettings, }, - retries::{FixedRetryPolicy, RetryLogic}, + retries::{FibonacciRetryPolicy, JitterMode, RetryLogic}, service::map::MapLayer, sink::Response, Batch, BatchSink, Partition, PartitionBatchSink, @@ -37,13 +37,14 @@ mod health; mod map; pub mod net; -pub type Svc = RateLimit, Timeout>, L>>; +pub type Svc = + RateLimit, Timeout>, L>>; pub type TowerBatchedSink = BatchSink, B>; pub type TowerPartitionSink = PartitionBatchSink, B, K>; // Distributed service types pub type DistributedService = RateLimit< - Retry, Buffer, Req>, Req>>, + Retry, Buffer, Req>, Req>>, >; pub type DiscoveryService = BoxStream<'static, Result>, crate::Error>>; @@ -83,17 +84,34 @@ impl ServiceBuilderExt for ServiceBuilder { } } +pub trait TowerRequestConfigDefaults { + const CONCURRENCY: Concurrency = Concurrency::Adaptive; + const TIMEOUT_SECS: u64 = 60; + const RATE_LIMIT_DURATION_SECS: u64 = 1; + const RATE_LIMIT_NUM: u64 = i64::max_value() as u64; // i64 avoids TOML deserialize issue + const RETRY_ATTEMPTS: usize = isize::max_value() as usize; // isize avoids TOML deserialize issue + const RETRY_MAX_DURATION_SECS: u64 = 30; + const RETRY_INITIAL_BACKOFF_SECS: u64 = 1; +} + +#[derive(Clone, Copy, Debug)] +pub struct GlobalTowerRequestConfigDefaults; + +impl TowerRequestConfigDefaults for GlobalTowerRequestConfigDefaults {} + /// Middleware settings for outbound requests. /// -/// Various settings can be configured, such as concurrency and rate limits, timeouts, etc. +/// Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. +/// +/// Note that the retry backoff policy follows the Fibonacci sequence. #[serde_as] #[configurable_component] #[configurable(metadata(docs::advanced))] #[derive(Clone, Copy, Debug)] -pub struct TowerRequestConfig { +pub struct TowerRequestConfig { #[configurable(derived)] - #[serde(default = "default_concurrency")] - #[serde(skip_serializing_if = "concurrency_is_adaptive")] + #[serde(default = "default_concurrency::")] + #[serde(skip_serializing_if = "concurrency_is_default::")] pub concurrency: Concurrency, /// The time a request can take before being aborted. @@ -101,170 +119,116 @@ pub struct TowerRequestConfig { /// Datadog highly recommends that you do not lower this value below the service's internal timeout, as this could /// create orphaned requests, pile on retries, and result in duplicate data downstream. #[configurable(metadata(docs::type_unit = "seconds"))] - #[serde(default = "default_timeout_secs")] #[configurable(metadata(docs::human_name = "Timeout"))] - pub timeout_secs: Option, + #[serde(default = "default_timeout_secs::")] + pub timeout_secs: u64, /// The time window used for the `rate_limit_num` option. #[configurable(metadata(docs::type_unit = "seconds"))] - #[serde(default = "default_rate_limit_duration_secs")] #[configurable(metadata(docs::human_name = "Rate Limit Duration"))] - pub rate_limit_duration_secs: Option, + #[serde(default = "default_rate_limit_duration_secs::")] + pub rate_limit_duration_secs: u64, /// The maximum number of requests allowed within the `rate_limit_duration_secs` time window. #[configurable(metadata(docs::type_unit = "requests"))] - #[serde(default = "default_rate_limit_num")] #[configurable(metadata(docs::human_name = "Rate Limit Number"))] - pub rate_limit_num: Option, + #[serde(default = "default_rate_limit_num::")] + pub rate_limit_num: u64, /// The maximum number of retries to make for failed requests. - /// - /// The default, for all intents and purposes, represents an infinite number of retries. #[configurable(metadata(docs::type_unit = "retries"))] - #[serde(default = "default_retry_attempts")] - pub retry_attempts: Option, + #[serde(default = "default_retry_attempts::")] + pub retry_attempts: usize, /// The maximum amount of time to wait between retries. #[configurable(metadata(docs::type_unit = "seconds"))] - #[serde(default = "default_retry_max_duration_secs")] #[configurable(metadata(docs::human_name = "Max Retry Duration"))] - pub retry_max_duration_secs: Option, + #[serde(default = "default_retry_max_duration_secs::")] + pub retry_max_duration_secs: u64, /// The amount of time to wait before attempting the first retry for a failed request. /// /// After the first retry has failed, the fibonacci sequence is used to select future backoffs. #[configurable(metadata(docs::type_unit = "seconds"))] - #[serde(default = "default_retry_initial_backoff_secs")] #[configurable(metadata(docs::human_name = "Retry Initial Backoff"))] - pub retry_initial_backoff_secs: Option, + #[serde(default = "default_retry_initial_backoff_secs::")] + pub retry_initial_backoff_secs: u64, + + #[configurable(derived)] + #[serde(default)] + pub retry_jitter_mode: JitterMode, #[configurable(derived)] #[serde(default)] pub adaptive_concurrency: AdaptiveConcurrencySettings, + + #[serde(skip)] + pub _d: PhantomData, +} + +const fn default_concurrency() -> Concurrency { + D::CONCURRENCY } -const fn default_concurrency() -> Concurrency { - Concurrency::Adaptive +fn concurrency_is_default(concurrency: &Concurrency) -> bool { + *concurrency == D::CONCURRENCY } -const fn default_timeout_secs() -> Option { - Some(60) +const fn default_timeout_secs() -> u64 { + D::TIMEOUT_SECS } -const fn default_rate_limit_duration_secs() -> Option { - Some(1) +const fn default_rate_limit_duration_secs() -> u64 { + D::RATE_LIMIT_DURATION_SECS } -const fn default_rate_limit_num() -> Option { - // i64 avoids TOML deserialize issue - Some(i64::max_value() as u64) +const fn default_rate_limit_num() -> u64 { + D::RATE_LIMIT_NUM } -const fn default_retry_attempts() -> Option { - // i64 avoids TOML deserialize issue - Some(isize::max_value() as usize) +const fn default_retry_attempts() -> usize { + D::RETRY_ATTEMPTS } -const fn default_retry_max_duration_secs() -> Option { - Some(3_600) +const fn default_retry_max_duration_secs() -> u64 { + D::RETRY_MAX_DURATION_SECS } -const fn default_retry_initial_backoff_secs() -> Option { - Some(1) +const fn default_retry_initial_backoff_secs() -> u64 { + D::RETRY_INITIAL_BACKOFF_SECS } -impl Default for TowerRequestConfig { +impl Default for TowerRequestConfig { fn default() -> Self { Self { - concurrency: default_concurrency(), - timeout_secs: default_timeout_secs(), - rate_limit_duration_secs: default_rate_limit_duration_secs(), - rate_limit_num: default_rate_limit_num(), - retry_attempts: default_retry_attempts(), - retry_max_duration_secs: default_retry_max_duration_secs(), - retry_initial_backoff_secs: default_retry_initial_backoff_secs(), + concurrency: default_concurrency::(), + timeout_secs: default_timeout_secs::(), + rate_limit_duration_secs: default_rate_limit_duration_secs::(), + rate_limit_num: default_rate_limit_num::(), + retry_attempts: default_retry_attempts::(), + retry_max_duration_secs: default_retry_max_duration_secs::(), + retry_initial_backoff_secs: default_retry_initial_backoff_secs::(), adaptive_concurrency: AdaptiveConcurrencySettings::default(), - } - } -} + retry_jitter_mode: JitterMode::default(), -impl TowerRequestConfig { - pub fn new(concurrency: Concurrency) -> Self { - Self { - concurrency, - ..Default::default() + _d: PhantomData, } } +} - pub const fn timeout_secs(mut self, timeout_secs: u64) -> Self { - self.timeout_secs = Some(timeout_secs); - self - } - - pub const fn rate_limit_duration_secs(mut self, rate_limit_duration_secs: u64) -> Self { - self.rate_limit_duration_secs = Some(rate_limit_duration_secs); - self - } - - pub const fn rate_limit_num(mut self, rate_limit_num: u64) -> Self { - self.rate_limit_num = Some(rate_limit_num); - self - } - - pub const fn retry_attempts(mut self, retry_attempts: usize) -> Self { - self.retry_attempts = Some(retry_attempts); - self - } - - pub const fn retry_max_duration_secs(mut self, retry_max_duration_secs: u64) -> Self { - self.retry_max_duration_secs = Some(retry_max_duration_secs); - self - } - - pub const fn retry_initial_backoff_secs(mut self, retry_initial_backoff_secs: u64) -> Self { - self.retry_initial_backoff_secs = Some(retry_initial_backoff_secs); - self - } - - pub fn unwrap_with(&self, defaults: &Self) -> TowerRequestSettings { +impl TowerRequestConfig { + pub const fn into_settings(&self) -> TowerRequestSettings { // the unwrap() calls below are safe because the final defaults are always Some<> TowerRequestSettings { - concurrency: self.concurrency.parse_concurrency(defaults.concurrency), - timeout: Duration::from_secs( - self.timeout_secs - .or(defaults.timeout_secs) - .or(default_timeout_secs()) - .unwrap(), - ), - rate_limit_duration: Duration::from_secs( - self.rate_limit_duration_secs - .or(defaults.rate_limit_duration_secs) - .or(default_rate_limit_duration_secs()) - .unwrap(), - ), - rate_limit_num: self - .rate_limit_num - .or(defaults.rate_limit_num) - .or(default_rate_limit_num()) - .unwrap(), - retry_attempts: self - .retry_attempts - .or(defaults.retry_attempts) - .or(default_retry_attempts()) - .unwrap(), - retry_max_duration_secs: Duration::from_secs( - self.retry_max_duration_secs - .or(defaults.retry_max_duration_secs) - .or(default_retry_max_duration_secs()) - .unwrap(), - ), - retry_initial_backoff_secs: Duration::from_secs( - self.retry_initial_backoff_secs - .or(defaults.retry_initial_backoff_secs) - .or(default_retry_initial_backoff_secs()) - .unwrap(), - ), + concurrency: self.concurrency.parse_concurrency(), + timeout: Duration::from_secs(self.timeout_secs), + rate_limit_duration: Duration::from_secs(self.rate_limit_duration_secs), + rate_limit_num: self.rate_limit_num, + retry_attempts: self.retry_attempts, + retry_max_duration: Duration::from_secs(self.retry_max_duration_secs), + retry_initial_backoff: Duration::from_secs(self.retry_initial_backoff_secs), adaptive_concurrency: self.adaptive_concurrency, + retry_jitter_mode: self.retry_jitter_mode, } } } @@ -276,18 +240,20 @@ pub struct TowerRequestSettings { pub rate_limit_duration: Duration, pub rate_limit_num: u64, pub retry_attempts: usize, - pub retry_max_duration_secs: Duration, - pub retry_initial_backoff_secs: Duration, + pub retry_max_duration: Duration, + pub retry_initial_backoff: Duration, pub adaptive_concurrency: AdaptiveConcurrencySettings, + pub retry_jitter_mode: JitterMode, } impl TowerRequestSettings { - pub const fn retry_policy(&self, logic: L) -> FixedRetryPolicy { - FixedRetryPolicy::new( + pub fn retry_policy(&self, logic: L) -> FibonacciRetryPolicy { + FibonacciRetryPolicy::new( self.retry_attempts, - self.retry_initial_backoff_secs, - self.retry_max_duration_secs, + self.retry_initial_backoff, + self.retry_max_duration, logic, + self.retry_jitter_mode, ) } @@ -456,7 +422,7 @@ mod tests { #[test] fn concurrency_param_works() { - let cfg = TowerRequestConfig::default(); + let cfg = TowerRequestConfig::::default(); let toml = toml::to_string(&cfg).unwrap(); toml::from_str::(&toml).expect("Default config failed"); @@ -471,6 +437,10 @@ mod tests { .expect("Adaptive concurrency setting failed"); assert_eq!(cfg.concurrency, Concurrency::Adaptive); + let cfg = toml::from_str::(r#"concurrency = "none""#) + .expect("None concurrency setting failed"); + assert_eq!(cfg.concurrency, Concurrency::None); + toml::from_str::(r#"concurrency = "broken""#) .expect_err("Invalid concurrency setting didn't fail"); @@ -482,19 +452,82 @@ mod tests { } #[test] - fn config_merging_defaults_concurrency_to_none_if_unset() { - let cfg = TowerRequestConfig::default().unwrap_with(&TowerRequestConfig::default()); + fn into_settings_with_global_defaults() { + let cfg = TowerRequestConfig::::default(); + let settings = cfg.into_settings(); + + assert_eq!(settings.concurrency, None); + assert_eq!(settings.timeout, Duration::from_secs(60)); + assert_eq!(settings.rate_limit_duration, Duration::from_secs(1)); + assert_eq!(settings.rate_limit_num, i64::max_value() as u64); + assert_eq!(settings.retry_attempts, isize::max_value() as usize); + assert_eq!(settings.retry_max_duration, Duration::from_secs(30)); + assert_eq!(settings.retry_initial_backoff, Duration::from_secs(1)); + } + + #[derive(Clone, Copy, Debug)] + pub struct TestTowerRequestConfigDefaults; + + impl TowerRequestConfigDefaults for TestTowerRequestConfigDefaults { + const CONCURRENCY: Concurrency = Concurrency::None; + const TIMEOUT_SECS: u64 = 1; + const RATE_LIMIT_DURATION_SECS: u64 = 2; + const RATE_LIMIT_NUM: u64 = 3; + const RETRY_ATTEMPTS: usize = 4; + const RETRY_MAX_DURATION_SECS: u64 = 5; + const RETRY_INITIAL_BACKOFF_SECS: u64 = 6; + } + + #[test] + fn into_settings_with_overridden_defaults() { + let cfg = TowerRequestConfig::::default(); + let settings = cfg.into_settings(); + + assert_eq!(settings.concurrency, Some(1)); + assert_eq!(settings.timeout, Duration::from_secs(1)); + assert_eq!(settings.rate_limit_duration, Duration::from_secs(2)); + assert_eq!(settings.rate_limit_num, 3); + assert_eq!(settings.retry_attempts, 4); + assert_eq!(settings.retry_max_duration, Duration::from_secs(5)); + assert_eq!(settings.retry_initial_backoff, Duration::from_secs(6)); + } - assert_eq!(cfg.concurrency, None); + #[test] + fn into_settings_with_populated_config() { + // Populate with values not equal to the global defaults. + let cfg = toml::from_str::( + r#" concurrency = 16 + timeout_secs = 1 + rate_limit_duration_secs = 2 + rate_limit_num = 3 + retry_attempts = 4 + retry_max_duration_secs = 5 + retry_initial_backoff_secs = 6 + "#, + ) + .expect("Config failed to parse"); + + // Merge with defaults + let settings = cfg.into_settings(); + assert_eq!( + settings.concurrency, + Concurrency::Fixed(16).parse_concurrency() + ); + assert_eq!(settings.timeout, Duration::from_secs(1)); + assert_eq!(settings.rate_limit_duration, Duration::from_secs(2)); + assert_eq!(settings.rate_limit_num, 3); + assert_eq!(settings.retry_attempts, 4); + assert_eq!(settings.retry_max_duration, Duration::from_secs(5)); + assert_eq!(settings.retry_initial_backoff, Duration::from_secs(6)); } #[tokio::test] async fn partition_sink_retry_concurrency() { - let cfg = TowerRequestConfig { + let cfg: TowerRequestConfig = TowerRequestConfig { concurrency: Concurrency::Fixed(1), ..TowerRequestConfig::default() }; - let settings = cfg.unwrap_with(&TowerRequestConfig::default()); + let settings = cfg.into_settings(); let sent_requests = Arc::new(Mutex::new(Vec::new())); diff --git a/src/sinks/util/service/concurrency.rs b/src/sinks/util/service/concurrency.rs index cc77ef0338fc0c..6bb9586ccc7f64 100644 --- a/src/sinks/util/service/concurrency.rs +++ b/src/sinks/util/service/concurrency.rs @@ -56,26 +56,15 @@ impl Default for Concurrency { } impl Concurrency { - const fn if_adaptive(self, other: Self) -> Self { + pub const fn parse_concurrency(&self) -> Option { match self { - Self::Adaptive => other, - _ => self, - } - } - - pub const fn parse_concurrency(&self, other: Self) -> Option { - match self.if_adaptive(other) { Concurrency::None => Some(1), Concurrency::Adaptive => None, - Concurrency::Fixed(limit) => Some(limit), + Concurrency::Fixed(limit) => Some(*limit), } } } -pub const fn concurrency_is_adaptive(concurrency: &Concurrency) -> bool { - matches!(concurrency, Concurrency::Adaptive) -} - impl<'de> Deserialize<'de> for Concurrency { // Deserialize either a positive integer or the string "adaptive" fn deserialize(deserializer: D) -> Result diff --git a/src/sinks/vector/config.rs b/src/sinks/vector/config.rs index 095eda4a4503c9..9f2664c3a11b3e 100644 --- a/src/sinks/vector/config.rs +++ b/src/sinks/vector/config.rs @@ -123,7 +123,7 @@ impl SinkConfig for VectorConfig { let healthcheck_client = VectorService::new(client.clone(), healthcheck_uri, false); let healthcheck = healthcheck(healthcheck_client, cx.healthcheck); let service = VectorService::new(client, uri, self.compression); - let request_settings = self.request.unwrap_with(&TowerRequestConfig::default()); + let request_settings = self.request.into_settings(); let batch_settings = self.batch.into_batcher_settings()?; let service = ServiceBuilder::new() diff --git a/src/sources/datadog_agent/mod.rs b/src/sources/datadog_agent/mod.rs index ba4ac56f7997a5..eaedf2be790455 100644 --- a/src/sources/datadog_agent/mod.rs +++ b/src/sources/datadog_agent/mod.rs @@ -87,20 +87,20 @@ pub struct DatadogAgentConfig { #[serde(default = "crate::serde::default_false")] disable_logs: bool, - /// If this is set to `true`, metrics are not accepted by the component. + /// If this is set to `true`, metrics (beta) are not accepted by the component. #[configurable(metadata(docs::advanced))] #[serde(default = "crate::serde::default_false")] disable_metrics: bool, - /// If this is set to `true`, traces are not accepted by the component. + /// If this is set to `true`, traces (alpha) are not accepted by the component. #[configurable(metadata(docs::advanced))] #[serde(default = "crate::serde::default_false")] disable_traces: bool, - /// If this is set to `true` logs, metrics, and traces are sent to different outputs. + /// If this is set to `true`, logs, metrics (beta), and traces (alpha) are sent to different outputs. /// /// - /// For a source component named `agent`, the received logs, metrics, and traces can then be + /// For a source component named `agent`, the received logs, metrics (beta), and traces (alpha) can then be /// configured as input to other components by specifying `agent.logs`, `agent.metrics`, and /// `agent.traces`, respectively. #[configurable(metadata(docs::advanced))] diff --git a/src/sources/file.rs b/src/sources/file.rs index 11656e73193389..c8a670141a6a65 100644 --- a/src/sources/file.rs +++ b/src/sources/file.rs @@ -859,6 +859,9 @@ mod tests { }, data_dir: Some(dir.path().to_path_buf()), glob_minimum_cooldown_ms: Duration::from_millis(100), + internal_metrics: FileInternalMetricsConfig { + include_file_tag: true, + }, ..Default::default() } } diff --git a/src/sources/kafka.rs b/src/sources/kafka.rs index f53a3188e5410b..08313dde581d2e 100644 --- a/src/sources/kafka.rs +++ b/src/sources/kafka.rs @@ -81,7 +81,7 @@ enum BuildError { SubscribeError { source: rdkafka::error::KafkaError }, } -/// Metrics configuration. +/// Metrics (beta) configuration. #[configurable_component] #[derive(Clone, Debug, Default)] struct Metrics { diff --git a/src/template.rs b/src/template.rs index 3eccf147126849..136a2bd3d25471 100644 --- a/src/template.rs +++ b/src/template.rs @@ -4,7 +4,7 @@ use std::{borrow::Cow, convert::TryFrom, fmt, hash::Hash, path::PathBuf}; use bytes::Bytes; use chrono::{ format::{strftime::StrftimeItems, Item}, - Utc, + FixedOffset, Utc, }; use once_cell::sync::Lazy; use regex::Regex; @@ -63,6 +63,9 @@ pub struct Template { #[serde(skip)] reserve_size: usize, + + #[serde(skip)] + tz_offset: Option, } impl TryFrom<&str> for Template { @@ -116,6 +119,7 @@ impl TryFrom> for Template { src: src.into_owned(), is_static, reserve_size, + tz_offset: None, } }) } @@ -137,6 +141,11 @@ impl fmt::Display for Template { impl ConfigurableString for Template {} impl Template { + /// set tz offset + pub const fn with_tz_offset(mut self, tz_offset: Option) -> Self { + self.tz_offset = tz_offset; + self + } /// Renders the given template with data from the event. pub fn render<'a>( &self, @@ -163,7 +172,9 @@ impl Template { for part in &self.parts { match part { Part::Literal(lit) => out.push_str(lit), - Part::Strftime(items) => out.push_str(&render_timestamp(items, event)), + Part::Strftime(items) => { + out.push_str(&render_timestamp(items, event, self.tz_offset)) + } Part::Reference(key) => { out.push_str( &match event { @@ -344,8 +355,12 @@ fn render_metric_field<'a>(key: &str, metric: &'a Metric) -> Option<&'a str> { } } -fn render_timestamp(items: &ParsedStrftime, event: EventRef<'_>) -> String { - match event { +fn render_timestamp( + items: &ParsedStrftime, + event: EventRef<'_>, + tz_offset: Option, +) -> String { + let timestamp = match event { EventRef::Log(log) => log_schema() .timestamp_key_target_path() .and_then(|timestamp_key| { @@ -365,14 +380,24 @@ fn render_timestamp(items: &ParsedStrftime, event: EventRef<'_>) -> String { }) } } - .unwrap_or_else(Utc::now) - .format_with_items(items.as_items()) - .to_string() + .unwrap_or_else(Utc::now); + + match tz_offset { + Some(offset) => timestamp + .with_timezone(&offset) + .format_with_items(items.as_items()) + .to_string(), + None => timestamp + .with_timezone(&chrono::Utc) + .format_with_items(items.as_items()) + .to_string(), + } } #[cfg(test)] mod tests { - use chrono::TimeZone; + use chrono::{Offset, TimeZone, Utc}; + use chrono_tz::Tz; use vector_lib::lookup::{metadata_path, PathPrefix}; use vector_lib::metric_tags; @@ -654,6 +679,25 @@ mod tests { ); } + #[test] + fn render_log_with_timezone() { + let ts = Utc.with_ymd_and_hms(2001, 2, 3, 4, 5, 6).unwrap(); + + let template = Template::try_from("vector-%Y-%m-%d-%H.log").unwrap(); + let mut event = Event::Log(LogEvent::from("hello world")); + event.as_mut_log().insert( + (PathPrefix::Event, log_schema().timestamp_key().unwrap()), + ts, + ); + + let tz: Tz = "Asia/Singapore".parse().unwrap(); + let offset = Some(Utc::now().with_timezone(&tz).offset().fix()); + assert_eq!( + Ok(Bytes::from("vector-2001-02-03-12.log")), + template.with_tz_offset(offset).render(&event) + ); + } + fn sample_metric() -> Metric { Metric::new( "a-counter", diff --git a/src/transforms/aggregate.rs b/src/transforms/aggregate.rs index a3a1b1e11c2df5..f838b192217db9 100644 --- a/src/transforms/aggregate.rs +++ b/src/transforms/aggregate.rs @@ -24,7 +24,7 @@ use crate::{ pub struct AggregateConfig { /// The interval between flushes, in milliseconds. /// - /// During this time frame, metrics with the same series data (name, namespace, tags, and so on) are aggregated. + /// During this time frame, metrics (beta) with the same series data (name, namespace, tags, and so on) are aggregated. #[serde(default = "default_interval_ms")] #[configurable(metadata(docs::human_name = "Flush Interval"))] pub interval_ms: u64, diff --git a/src/transforms/log_to_metric.rs b/src/transforms/log_to_metric.rs index 5462ee1db8b561..a81b39d725bb98 100644 --- a/src/transforms/log_to_metric.rs +++ b/src/transforms/log_to_metric.rs @@ -4,13 +4,20 @@ use std::{collections::HashMap, num::ParseFloatError}; use chrono::Utc; use indexmap::IndexMap; use vector_lib::configurable::configurable_component; -use vector_lib::{config::LogNamespace, event::DatadogMetricOriginMetadata, event::{metric::Sample, metric::{Bucket, Quantile}}}; use vector_lib::event::LogEvent; -use vrl::path::parse_target_path; -use vrl::event_path; +use vector_lib::{ + config::LogNamespace, + event::DatadogMetricOriginMetadata, + event::{ + metric::Sample, + metric::{Bucket, Quantile}, + }, +}; +use vrl::path::{parse_target_path, PathParseError}; +use vrl::{event_path, path}; use crate::config::schema::Definition; -use crate::transforms::log_to_metric::TransformError::FieldNotFound; +use crate::transforms::log_to_metric::TransformError::PathNotFound; use crate::{ config::{ DataType, GenerateConfig, Input, OutputId, TransformConfig, TransformContext, @@ -21,8 +28,9 @@ use crate::{ Event, Value, }, internal_events::{ - MetricMetadataMetricDetailsNotFoundError, MetricMetadataInvalidFieldValueError, MetricMetadataParseFloatError, MetricMetadataParseArrayError, MetricMetadataParseIntError, - LogToMetricFieldNullError, LogToMetricParseFloatError, ParserMissingFieldError, DROP_EVENT, + LogToMetricFieldNullError, LogToMetricParseFloatError, + MetricMetadataInvalidFieldValueError, MetricMetadataMetricDetailsNotFoundError, + MetricMetadataParseError, ParserMissingFieldError, DROP_EVENT, }, schema, template::{Template, TemplateRenderingError}, @@ -38,7 +46,32 @@ const ORIGIN_SERVICE_VALUE: u32 = 3; pub struct LogToMetricConfig { /// A list of metrics to generate. pub metrics: Vec, - /// this flag will process all metrics to logs, if defined the `metrics` field is ignored. + /// Setting this flag changes the behavior of this transformation.
+ ///

Notably the `metrics` field will be ignored.

+ ///

All incoming events will be processed and if possible they will be converted to log events. + /// Otherwise, only items specified in the 'metrics' field will be processed.

+ /// 
use serde_json::json;
+    /// let json_event = json!({
+    ///     "counter": {
+    ///         "value": 10.0
+    ///     },
+    ///     "kind": "incremental",
+    ///     "name": "test.transform.counter",
+    ///     "tags": {
+    ///         "env": "test_env",
+    ///         "host": "localhost"
+    ///     }
+    /// });
+    ///
+ /// + /// This is an example JSON representation of a counter with the following properties: + /// + /// - `counter`: An object with a single property `value` representing the counter value, in this case, `10.0`). + /// - `kind`: A string indicating the kind of counter, in this case, "incremental". + /// - `name`: A string representing the name of the counter, here set to "test.transform.counter". + /// - `tags`: An object containing additional tags such as "env" and "host". + /// + /// Objects that can be processed include counter, histogram, gauge, set and summary. pub all_metrics: Option, } @@ -154,7 +187,7 @@ impl GenerateConfig for LogToMetricConfig { }], all_metrics: Some(true), }) - .unwrap() + .unwrap() } } @@ -190,31 +223,44 @@ impl LogToMetric { } } +/// Kinds of TranformError for Parsing +#[configurable_component] +#[derive(Clone, Debug)] +pub enum TransformParseErrorKind { + /// Error when Parsing a Float + FloatError, + /// Error when Parsing an Int + IntError, + /// Errors when Parsing Arrays + ArrayError, +} + +impl std::fmt::Display for TransformParseErrorKind { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + enum TransformError { - FieldNotFound { - field: String, + PathNotFound { + path: String, }, - FieldNull { - field: String, + PathNull { + path: String, }, MetricDetailsNotFound, MetricValueError { - field: String, - field_value: String, + path: String, + path_value: String, }, - ParseMetricFloatError { - field: String, + ParseError { + path: String, + kind: TransformParseErrorKind, }, ParseFloatError { - field: String, + path: String, error: ParseFloatError, }, - ParseIntError { - field: String - }, - ParseArrayError { - field: String - }, TemplateRenderingError(TemplateRenderingError), } @@ -274,7 +320,7 @@ fn render_tag_into( Ok(()) } -fn to_metric(config: &MetricConfig, event: &Event) -> Result { +fn to_metric_with_config(config: &MetricConfig, event: &Event) -> Result { let log = event.as_log(); let timestamp = log @@ -294,16 +340,16 @@ fn to_metric(config: &MetricConfig, event: &Event) -> Result Err(TransformError::FieldNotFound { - field: field.to_string(), + None => Err(TransformError::PathNotFound { + path: field.to_string(), }), - Some(Value::Null) => Err(TransformError::FieldNull { - field: field.to_string(), + Some(Value::Null) => Err(TransformError::PathNull { + path: field.to_string(), }), Some(value) => Ok(value), }?; @@ -323,7 +369,7 @@ fn to_metric(config: &MetricConfig, event: &Event) -> Result Result { let value = value.to_string_lossy().parse().map_err(|error| { TransformError::ParseFloatError { - field: field.to_string(), + path: field.to_string(), error, } })?; @@ -352,7 +398,7 @@ fn to_metric(config: &MetricConfig, event: &Event) -> Result { let value = value.to_string_lossy().parse().map_err(|error| { TransformError::ParseFloatError { - field: field.to_string(), + path: field.to_string(), error, } })?; @@ -368,7 +414,7 @@ fn to_metric(config: &MetricConfig, event: &Event) -> Result { let value = value.to_string_lossy().parse().map_err(|error| { TransformError::ParseFloatError { - field: field.to_string(), + path: field.to_string(), error, } })?; @@ -399,79 +445,110 @@ fn bytes_to_str(value: &Value) -> Option { } } -fn get_str_from_log(log: &LogEvent, key: &str) -> Option { - log.get(event_path!(key)).and_then(bytes_to_str) +fn try_get_string_from_log(log: &LogEvent, path: &str) -> Result, TransformError> { + // TODO: update returned errors after `TransformError` is refactored. + let maybe_value = log.parse_path_and_get_value(path).map_err(|e| match e { + PathParseError::InvalidPathSyntax { path } => PathNotFound { + path: path.to_string(), + }, + })?; + match maybe_value { + None => Err(PathNotFound { + path: path.to_string(), + }), + Some(v) => Ok(bytes_to_str(v)), + } } fn get_counter_value(log: &LogEvent) -> Result { - let counter_value = log.get("counter.value") - .ok_or_else(|| TransformError::FieldNotFound { - field: "counter.value".to_string(), + let counter_value = log + .get(event_path!("counter", "value")) + .ok_or_else(|| TransformError::PathNotFound { + path: "counter.value".to_string(), })? .as_float() - .ok_or_else(|| TransformError::ParseMetricFloatError { - field: "counter.value".to_string(), + .ok_or_else(|| TransformError::ParseError { + path: "counter.value".to_string(), + kind: TransformParseErrorKind::FloatError, })?; - Ok(MetricValue::Counter { value: *counter_value }) + Ok(MetricValue::Counter { + value: *counter_value, + }) } fn get_gauge_value(log: &LogEvent) -> Result { - let gauge_value = log.get("gauge.value") - .ok_or_else(|| TransformError::FieldNotFound { - field: "gauge.value".to_string(), + let gauge_value = log + .get(event_path!("gauge", "value")) + .ok_or_else(|| TransformError::PathNotFound { + path: "gauge.value".to_string(), })? .as_float() - .ok_or_else(|| TransformError::ParseMetricFloatError { - field: "gauge.value".to_string(), + .ok_or_else(|| TransformError::ParseError { + path: "gauge.value".to_string(), + kind: TransformParseErrorKind::FloatError, })?; - Ok(MetricValue::Gauge { value: *gauge_value }) + Ok(MetricValue::Gauge { + value: *gauge_value, + }) } fn get_distribution_value(log: &LogEvent) -> Result { - let event_samples = log.get("distribution.samples") - .ok_or_else(|| TransformError::FieldNotFound { - field: "distribution.samples".to_string(), + let event_samples = log + .get(event_path!("distribution", "samples")) + .ok_or_else(|| TransformError::PathNotFound { + path: "distribution.samples".to_string(), })? .as_array() - .ok_or_else(|| TransformError::ParseArrayError { - field: "distribution.samples".to_string() + .ok_or_else(|| TransformError::ParseError { + path: "distribution.samples".to_string(), + kind: TransformParseErrorKind::ArrayError, })?; let mut samples: Vec = Vec::new(); for e_sample in event_samples { - let value = e_sample.get("value") - .ok_or_else(|| TransformError::FieldNotFound { - field: "value".to_string(), + let value = e_sample + .get(path!("value")) + .ok_or_else(|| TransformError::PathNotFound { + path: "value".to_string(), })? .as_float() - .ok_or_else(|| TransformError::ParseMetricFloatError { - field: "value".to_string(), + .ok_or_else(|| TransformError::ParseError { + path: "value".to_string(), + kind: TransformParseErrorKind::FloatError, })?; - let rate = e_sample.get("rate") - .ok_or_else(|| TransformError::FieldNotFound { - field: "rate".to_string(), + let rate = e_sample + .get(path!("rate")) + .ok_or_else(|| TransformError::PathNotFound { + path: "rate".to_string(), })? .as_integer() - .ok_or_else(|| TransformError::ParseIntError { - field: "rate".to_string() + .ok_or_else(|| TransformError::ParseError { + path: "rate".to_string(), + kind: TransformParseErrorKind::IntError, })?; - samples.push(Sample { value: *value, rate: rate as u32 }); + samples.push(Sample { + value: *value, + rate: rate as u32, + }); } - let statistic_str = get_str_from_log(&log, "distribution.statistic") - .ok_or_else(|| TransformError::FieldNotFound { - field: "distribution.statistic".to_string(), - })?; - + let statistic_str = match try_get_string_from_log(log, "distribution.statistic")? { + Some(n) => n, + None => { + return Err(TransformError::PathNotFound { + path: "distribution.statistic".to_string(), + }) + } + }; let statistic_kind = match statistic_str.as_str() { "histogram" => Ok(StatisticKind::Histogram), "summary" => Ok(StatisticKind::Summary), _ => Err(TransformError::MetricValueError { - field: "distribution.statistic".to_string(), - field_value: statistic_str.to_string(), + path: "distribution.statistic".to_string(), + path_value: statistic_str.to_string(), }), }?; @@ -482,59 +559,67 @@ fn get_distribution_value(log: &LogEvent) -> Result } fn get_histogram_value(log: &LogEvent) -> Result { - let event_buckets = log.get("histogram.buckets") - .ok_or_else(|| TransformError::FieldNotFound { - field: "histogram.buckets".to_string(), + let event_buckets = log + .get(event_path!("histogram", "buckets")) + .ok_or_else(|| TransformError::PathNotFound { + path: "histogram.buckets".to_string(), })? .as_array() - .ok_or_else(|| TransformError::ParseArrayError { - field: "histogram.buckets".to_string() + .ok_or_else(|| TransformError::ParseError { + path: "histogram.buckets".to_string(), + kind: TransformParseErrorKind::ArrayError, })?; let mut buckets: Vec = Vec::new(); for e_bucket in event_buckets { - let upper_limit = e_bucket.get("upper_limit") - .ok_or_else(|| TransformError::FieldNotFound { - field: "histogram.buckets.upper_limit".to_string(), + let upper_limit = e_bucket + .get(path!("upper_limit")) + .ok_or_else(|| TransformError::PathNotFound { + path: "histogram.buckets.upper_limit".to_string(), })? .as_float() - .ok_or_else(|| TransformError::ParseMetricFloatError { - field: "histogram.buckets.upper_limit".to_string(), + .ok_or_else(|| TransformError::ParseError { + path: "histogram.buckets.upper_limit".to_string(), + kind: TransformParseErrorKind::FloatError, })?; - let count = e_bucket.get("count") - .ok_or_else(|| TransformError::FieldNotFound { - field: "histogram.buckets.count".to_string(), + let count = e_bucket + .get(path!("count")) + .ok_or_else(|| TransformError::PathNotFound { + path: "histogram.buckets.count".to_string(), })? .as_integer() - .ok_or_else(|| TransformError::ParseIntError { - field: "histogram.buckets.count".to_string() + .ok_or_else(|| TransformError::ParseError { + path: "histogram.buckets.count".to_string(), + kind: TransformParseErrorKind::IntError, })?; - buckets.push( - Bucket { - upper_limit: *upper_limit, - count: count as u64, - } - ); + buckets.push(Bucket { + upper_limit: *upper_limit, + count: count as u64, + }); } - let count = log.get("histogram.count") - .ok_or_else(|| TransformError::FieldNotFound { - field: "histogram.count".to_string(), + let count = log + .get(event_path!("histogram", "count")) + .ok_or_else(|| TransformError::PathNotFound { + path: "histogram.count".to_string(), })? .as_integer() - .ok_or_else(|| TransformError::ParseIntError { - field: "histogram.count".to_string() + .ok_or_else(|| TransformError::ParseError { + path: "histogram.count".to_string(), + kind: TransformParseErrorKind::IntError, })?; - let sum = log.get("histogram.sum") - .ok_or_else(|| TransformError::FieldNotFound { - field: "histogram.sum".to_string(), + let sum = log + .get(event_path!("histogram", "sum")) + .ok_or_else(|| TransformError::PathNotFound { + path: "histogram.sum".to_string(), })? .as_float() - .ok_or_else(|| TransformError::ParseMetricFloatError { - field: "histogram.sum".to_string(), + .ok_or_else(|| TransformError::ParseError { + path: "histogram.sum".to_string(), + kind: TransformParseErrorKind::FloatError, })?; Ok(MetricValue::AggregatedHistogram { @@ -545,59 +630,67 @@ fn get_histogram_value(log: &LogEvent) -> Result { } fn get_summary_value(log: &LogEvent) -> Result { - let event_quantiles = log.get("summary.quantiles") - .ok_or_else(|| TransformError::FieldNotFound { - field: "summary.quantiles".to_string(), + let event_quantiles = log + .get(event_path!("summary", "quantiles")) + .ok_or_else(|| TransformError::PathNotFound { + path: "summary.quantiles".to_string(), })? .as_array() - .ok_or_else(|| TransformError::ParseArrayError { - field: "summary.quantiles".to_string() + .ok_or_else(|| TransformError::ParseError { + path: "summary.quantiles".to_string(), + kind: TransformParseErrorKind::ArrayError, })?; let mut quantiles: Vec = Vec::new(); for e_quantile in event_quantiles { - let quantile = e_quantile.get("quantile") - .ok_or_else(|| TransformError::FieldNotFound { - field: "summary.quantiles.quantile".to_string(), + let quantile = e_quantile + .get(path!("quantile")) + .ok_or_else(|| TransformError::PathNotFound { + path: "summary.quantiles.quantile".to_string(), })? .as_float() - .ok_or_else(|| TransformError::ParseMetricFloatError { - field: "summary.quantiles.quantile".to_string(), + .ok_or_else(|| TransformError::ParseError { + path: "summary.quantiles.quantile".to_string(), + kind: TransformParseErrorKind::FloatError, })?; - let value = e_quantile.get("value") - .ok_or_else(|| TransformError::FieldNotFound { - field: "summary.quantiles.value".to_string(), + let value = e_quantile + .get(path!("value")) + .ok_or_else(|| TransformError::PathNotFound { + path: "summary.quantiles.value".to_string(), })? .as_float() - .ok_or_else(|| TransformError::ParseMetricFloatError { - field: "summary.quantiles.value".to_string(), + .ok_or_else(|| TransformError::ParseError { + path: "summary.quantiles.value".to_string(), + kind: TransformParseErrorKind::FloatError, })?; - quantiles.push( - Quantile { - quantile: *quantile, - value: *value, - } - ) + quantiles.push(Quantile { + quantile: *quantile, + value: *value, + }) } - let count = log.get("summary.count") - .ok_or_else(|| TransformError::FieldNotFound { - field: "summary.count".to_string(), + let count = log + .get(event_path!("summary", "count")) + .ok_or_else(|| TransformError::PathNotFound { + path: "summary.count".to_string(), })? .as_integer() - .ok_or_else(|| TransformError::ParseIntError { - field: "summary.count".to_string() + .ok_or_else(|| TransformError::ParseError { + path: "summary.count".to_string(), + kind: TransformParseErrorKind::IntError, })?; - let sum = log.get("summary.sum") - .ok_or_else(|| TransformError::FieldNotFound { - field: "summary.sum".to_string(), + let sum = log + .get(event_path!("summary", "sum")) + .ok_or_else(|| TransformError::PathNotFound { + path: "summary.sum".to_string(), })? .as_float() - .ok_or_else(|| TransformError::ParseMetricFloatError { - field: "summary.sum".to_string(), + .ok_or_else(|| TransformError::ParseError { + path: "summary.sum".to_string(), + kind: TransformParseErrorKind::FloatError, })?; Ok(MetricValue::AggregatedSummary { @@ -607,7 +700,7 @@ fn get_summary_value(log: &LogEvent) -> Result { }) } -fn to_metrics_metadata(event: &Event) -> Result { +fn to_metrics(event: &Event) -> Result { let log = event.as_log(); let timestamp = log .get_timestamp() @@ -615,43 +708,54 @@ fn to_metrics_metadata(event: &Event) -> Result { .cloned() .or_else(|| Some(Utc::now())); - let name = match get_str_from_log(&log, "name") { + let name = match try_get_string_from_log(log, "name")? { Some(n) => n, - None => return Err(TransformError::FieldNotFound { - field: "name".to_string() - }), + None => { + return Err(TransformError::PathNotFound { + path: "name".to_string(), + }) + } }; let tags = &mut MetricTags::default(); - if let Some(els) = log.get("tags") { + if let Some(els) = log.get(event_path!("tags")) { if let Some(el) = els.as_object() { for (key, value) in el { - tags.insert(String::from(key).to_string(), bytes_to_str(value)); + tags.insert(String::from(key.as_ref()).to_string(), bytes_to_str(value)); } } } let tags_result = Some(tags.clone()); - let kind_str = get_str_from_log(&log, "kind").ok_or_else(|| TransformError::FieldNotFound { - field: "kind".to_string(), - })?; + let kind_str = match try_get_string_from_log(log, "kind")? { + Some(n) => n, + None => { + return Err(TransformError::PathNotFound { + path: "kind".to_string(), + }) + } + }; + let kind = match kind_str.as_str() { "absolute" => Ok(MetricKind::Absolute), "incremental" => Ok(MetricKind::Incremental), - value => Err(TransformError::MetricValueError { field: "kind".to_string(), field_value: value.to_string() }) + value => Err(TransformError::MetricValueError { + path: "kind".to_string(), + path_value: value.to_string(), + }), }?; let mut value: Option = None; if let Some(root_event) = log.as_map() { - for (key, _v) in root_event { + for key in root_event.keys() { value = match key.as_str() { - "gauge" => Some(get_gauge_value(&log)?), - "distribution" => Some(get_distribution_value(&log)?), - "histogram" => Some(get_histogram_value(&log)?), - "summary" => Some(get_summary_value(&log)?), - "counter" => Some(get_counter_value(&log)?), - _ => { None } + "gauge" => Some(get_gauge_value(log)?), + "distribution" => Some(get_distribution_value(log)?), + "histogram" => Some(get_histogram_value(log)?), + "summary" => Some(get_summary_value(log)?), + "counter" => Some(get_counter_value(log)?), + _ => None, }; if value.is_some() { @@ -662,50 +766,48 @@ fn to_metrics_metadata(event: &Event) -> Result { let value = value.ok_or(TransformError::MetricDetailsNotFound)?; - Ok(Metric::new_with_metadata(name, kind, value, log.metadata().clone()) - .with_namespace(get_str_from_log(log, "namespace")) - .with_tags(tags_result) - .with_timestamp(timestamp)) + Ok( + Metric::new_with_metadata(name, kind, value, log.metadata().clone()) + .with_namespace(try_get_string_from_log(log, "namespace")?) + .with_tags(tags_result) + .with_timestamp(timestamp), + ) } impl FunctionTransform for LogToMetric { fn transform(&mut self, output: &mut OutputBuffer, event: Event) { // Metrics are "all or none" for a specific log. If a single fails, none are produced. let mut buffer = Vec::with_capacity(self.config.metrics.len()); - if self.config.all_metrics.is_some_and(|all_metrics| all_metrics == true) { - match to_metrics_metadata(&event) { + if self + .config + .all_metrics + .is_some_and(|all_metrics| all_metrics) + { + match to_metrics(&event) { Ok(metric) => { output.push(Event::Metric(metric)); } Err(err) => { match err { - TransformError::MetricValueError { field, field_value } => emit!(MetricMetadataInvalidFieldValueError { - field: field.as_ref(), - field_value: field_value.as_ref() - }), - TransformError::FieldNotFound { field } => { - emit!(ParserMissingFieldError:: { - field: field.as_ref() + TransformError::MetricValueError { path, path_value } => { + emit!(MetricMetadataInvalidFieldValueError { + field: path.as_ref(), + field_value: path_value.as_ref() }) } - TransformError::ParseMetricFloatError { field } => { - emit!(MetricMetadataParseFloatError { - field: field.as_ref() - }) - } - TransformError::ParseIntError { field } => { - emit!(MetricMetadataParseIntError { - field: field.as_ref() + TransformError::PathNotFound { path } => { + emit!(ParserMissingFieldError:: { + field: path.as_ref() }) } - TransformError::ParseArrayError { field } => { - emit!(MetricMetadataParseArrayError { - field: field.as_ref() + TransformError::ParseError { path, kind } => { + emit!(MetricMetadataParseError { + field: path.as_ref(), + kind: &kind.to_string(), }) } TransformError::MetricDetailsNotFound {} => { - emit!(MetricMetadataMetricDetailsNotFoundError { - }) + emit!(MetricMetadataMetricDetailsNotFoundError {}) } _ => {} }; @@ -713,32 +815,34 @@ impl FunctionTransform for LogToMetric { } } else { for config in self.config.metrics.iter() { - match to_metric(config, &event) { + match to_metric_with_config(config, &event) { Ok(metric) => { buffer.push(Event::Metric(metric)); } Err(err) => { match err { - TransformError::FieldNull { field } => emit!(LogToMetricFieldNullError { - field: field.as_ref() - }), - TransformError::FieldNotFound { field } => { + TransformError::PathNull { path } => { + emit!(LogToMetricFieldNullError { + field: path.as_ref() + }) + } + TransformError::PathNotFound { path } => { emit!(ParserMissingFieldError:: { - field: field.as_ref() - }) + field: path.as_ref() + }) } - TransformError::ParseFloatError { field, error } => { + TransformError::ParseFloatError { path, error } => { emit!(LogToMetricParseFloatError { - field: field.as_ref(), - error - }) + field: path.as_ref(), + error + }) } TransformError::TemplateRenderingError(error) => { emit!(crate::internal_events::TemplateRenderingError { - error, - drop_event: true, - field: None, - }) + error, + drop_event: true, + field: None, + }) } _ => {} }; @@ -774,7 +878,7 @@ mod tests { use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use vector_lib::config::ComponentKey; - use vector_lib::event::{EventMetadata}; + use vector_lib::event::EventMetadata; use vector_lib::metric_tags; #[test] @@ -818,7 +922,7 @@ mod tests { assert_eq!(out.recv().await, None); result }) - .await + .await } async fn do_transform_multiple_events( @@ -846,7 +950,7 @@ mod tests { assert_eq!(out.recv().await, None); results }) - .await + .await } #[tokio::test] @@ -883,7 +987,7 @@ mod tests { MetricValue::Counter { value: 1.0 }, metadata, ) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } @@ -927,13 +1031,13 @@ mod tests { MetricValue::Counter { value: 1.0 }, metadata, ) - .with_namespace(Some("app")) - .with_tags(Some(metric_tags!( + .with_namespace(Some("app")) + .with_tags(Some(metric_tags!( "method" => "post", "code" => "200", "host" => "localhost", ))) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } @@ -1027,7 +1131,7 @@ mod tests { MetricValue::Counter { value: 1.0 }, metadata ) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } @@ -1082,7 +1186,7 @@ mod tests { MetricValue::Counter { value: 33.99 }, metadata, ) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } @@ -1124,7 +1228,7 @@ mod tests { MetricValue::Counter { value: 33.99 }, metadata, ) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } @@ -1166,7 +1270,7 @@ mod tests { MetricValue::Gauge { value: 123.0 }, metadata, ) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } @@ -1263,7 +1367,7 @@ mod tests { MetricValue::Counter { value: 1.0 }, metadata.clone(), ) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); assert_eq!( output[1].clone().into_metric(), @@ -1273,7 +1377,7 @@ mod tests { MetricValue::Counter { value: 1.0 }, metadata, ) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } @@ -1331,7 +1435,7 @@ mod tests { }, metadata.clone(), ) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); assert_eq!( output[1].as_metric(), @@ -1341,8 +1445,8 @@ mod tests { MetricValue::Counter { value: 1.0 }, metadata, ) - .with_namespace(Some("local")) - .with_timestamp(Some(ts())) + .with_namespace(Some("local")) + .with_timestamp(Some(ts())) ); } @@ -1384,7 +1488,7 @@ mod tests { }, metadata, ) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } @@ -1427,7 +1531,7 @@ mod tests { }, metadata ) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } @@ -1470,13 +1574,14 @@ mod tests { }, metadata ) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } // Metric Metadata Tests - fn create_log_event(json_str : &str) -> Event { - let mut log_value: Value = serde_json::from_str(&*json_str).expect("JSON was not well-formatted"); + fn create_log_event(json_str: &str) -> Event { + let mut log_value: Value = + serde_json::from_str(json_str).expect("JSON was not well-formatted"); log_value.insert("timestamp", ts()); log_value.insert("namespace", "test_namespace"); @@ -1489,12 +1594,10 @@ mod tests { #[tokio::test] async fn transform_gauge() { - let config = parse_config( + let config = parse_yaml_config( r#" - [[metrics]] - type = "gauge" - field = "value" - name = "test.transform.gauge" + metrics: [] + all_metrics: true "#, ); @@ -1519,21 +1622,23 @@ mod tests { MetricValue::Gauge { value: 990.0 }, metric.metadata().clone(), ) - .with_namespace(Some("test_namespace")) - .with_tags(Some(metric_tags!( + .with_namespace(Some("test_namespace")) + .with_tags(Some(metric_tags!( "env" => "test_env", "host" => "localhost", ))) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } #[tokio::test] async fn transform_histogram() { - let config = parse_config(r#" - metrics: - all_fields: true - "#); + let config = parse_yaml_config( + r#" + metrics: [] + all_metrics: true + "#, + ); let json_str = r#"{ "histogram": { @@ -1566,7 +1671,7 @@ mod tests { } }"#; let log = create_log_event(json_str); - let metric = do_transform(config,log.clone()).await.unwrap(); + let metric = do_transform(config, log.clone()).await.unwrap(); assert_eq!( *metric.as_metric(), Metric::new_with_metadata( @@ -1596,21 +1701,23 @@ mod tests { }, metric.metadata().clone(), ) - .with_namespace(Some("test_namespace")) - .with_tags(Some(metric_tags!( + .with_namespace(Some("test_namespace")) + .with_tags(Some(metric_tags!( "env" => "test_env", "host" => "localhost", ))) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } #[tokio::test] async fn transform_distribution_histogram() { - let config = parse_config(r#" - metrics: - all_fields: true - "#); + let config = parse_yaml_config( + r#" + metrics: [] + all_metrics: true + "#, + ); let json_str = r#"{ "distribution": { @@ -1642,28 +1749,36 @@ mod tests { MetricKind::Absolute, MetricValue::Distribution { samples: vec![ - Sample { value: 1.0, rate: 1 }, - Sample { value: 2.0, rate: 2 }, + Sample { + value: 1.0, + rate: 1 + }, + Sample { + value: 2.0, + rate: 2 + }, ], statistic: StatisticKind::Histogram, }, metric.metadata().clone(), ) - .with_namespace(Some("test_namespace")) - .with_tags(Some(metric_tags!( + .with_namespace(Some("test_namespace")) + .with_tags(Some(metric_tags!( "env" => "test_env", "host" => "localhost", ))) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } #[tokio::test] async fn transform_distribution_summary() { - let config = parse_config(r#" - metrics: - all_fields: true - "#); + let config = parse_yaml_config( + r#" + metrics: [] + all_metrics: true + "#, + ); let json_str = r#"{ "distribution": { @@ -1687,7 +1802,7 @@ mod tests { } }"#; let log = create_log_event(json_str); - let metric = do_transform(config,log.clone()).await.unwrap(); + let metric = do_transform(config, log.clone()).await.unwrap(); assert_eq!( *metric.as_metric(), Metric::new_with_metadata( @@ -1695,28 +1810,36 @@ mod tests { MetricKind::Absolute, MetricValue::Distribution { samples: vec![ - Sample { value: 1.0, rate: 1 }, - Sample { value: 2.0, rate: 2 }, + Sample { + value: 1.0, + rate: 1 + }, + Sample { + value: 2.0, + rate: 2 + }, ], statistic: StatisticKind::Summary, }, metric.metadata().clone(), ) - .with_namespace(Some("test_namespace")) - .with_tags(Some(metric_tags!( + .with_namespace(Some("test_namespace")) + .with_tags(Some(metric_tags!( "env" => "test_env", "host" => "localhost", ))) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } #[tokio::test] async fn transform_summary() { - let config = parse_config(r#" - metrics: - all_fields: true - "#); + let config = parse_yaml_config( + r#" + metrics: [] + all_metrics: true + "#, + ); let json_str = r#"{ "summary": { @@ -1741,7 +1864,7 @@ mod tests { } }"#; let log = create_log_event(json_str); - let metric = do_transform(config,log.clone()).await.unwrap(); + let metric = do_transform(config, log.clone()).await.unwrap(); assert_eq!( *metric.as_metric(), Metric::new_with_metadata( @@ -1763,21 +1886,23 @@ mod tests { }, metric.metadata().clone(), ) - .with_namespace(Some("test_namespace")) - .with_tags(Some(metric_tags!( + .with_namespace(Some("test_namespace")) + .with_tags(Some(metric_tags!( "env" => "test_env", "host" => "localhost", ))) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } #[tokio::test] async fn transform_counter() { - let config = parse_config(r#" - metrics: - all_fields: true - "#); + let config = parse_yaml_config( + r#" + metrics: [] + all_metrics: true + "#, + ); let json_str = r#"{ "counter": { @@ -1791,7 +1916,7 @@ mod tests { } }"#; let log = create_log_event(json_str); - let metric = do_transform(config,log.clone()).await.unwrap(); + let metric = do_transform(config, log.clone()).await.unwrap(); assert_eq!( *metric.as_metric(), Metric::new_with_metadata( @@ -1800,13 +1925,12 @@ mod tests { MetricValue::Counter { value: 10.0 }, metric.metadata().clone(), ) - .with_namespace(Some("test_namespace")) - .with_tags(Some(metric_tags!( + .with_namespace(Some("test_namespace")) + .with_tags(Some(metric_tags!( "env" => "test_env", "host" => "localhost", ))) - .with_timestamp(Some(ts())) + .with_timestamp(Some(ts())) ); } - } diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 82e701833dda2a..d942e5d5af8127 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -11,7 +11,7 @@ publish = false anyhow = "1.0.75" cached = "0.46.1" chrono = { version = "0.4.31", default-features = false, features = ["serde", "clock"] } -clap = { version = "4.4.7", features = ["derive"] } +clap = { version = "4.4.8", features = ["derive"] } clap-verbosity-flag = "2.1.0" clap_complete = "4.4.4" confy = "0.5.1" diff --git a/website/content/en/docs/setup/installation/package-managers/apt.md b/website/content/en/docs/setup/installation/package-managers/apt.md index 45180397dafbfc..396961f906b8a9 100644 --- a/website/content/en/docs/setup/installation/package-managers/apt.md +++ b/website/content/en/docs/setup/installation/package-managers/apt.md @@ -19,7 +19,7 @@ Our APT repositories are provided by [Cloudsmith] and you can find [instructions First, add the Vector repo: ```shell -bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_vector0.sh)" +bash -c "$(curl -L https://setup.vector.dev)" ``` Then you can install the `vector` package: diff --git a/website/content/en/docs/setup/installation/package-managers/yum.md b/website/content/en/docs/setup/installation/package-managers/yum.md index d2fc334e601b5d..769ada412d23d4 100644 --- a/website/content/en/docs/setup/installation/package-managers/yum.md +++ b/website/content/en/docs/setup/installation/package-managers/yum.md @@ -13,7 +13,7 @@ Our Yum repositories are provided by [Cloudsmith] and you can find [instructions Add the repo: ```shell -bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_vector0.sh)" +bash -c "$(curl -L https://setup.vector.dev)" ``` Then you can install Vector: diff --git a/website/content/en/highlights/2023-11-07-new-linux-repos.md b/website/content/en/highlights/2023-11-07-new-linux-repos.md index afcb0c7f274fe3..83ad54738fca4c 100644 --- a/website/content/en/highlights/2023-11-07-new-linux-repos.md +++ b/website/content/en/highlights/2023-11-07-new-linux-repos.md @@ -38,7 +38,7 @@ The following command **removes** the existing repository and configures the new repository. ```sh -CSM_MIGRATE=true bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_vector0.sh)" +CSM_MIGRATE=true bash -c "$(curl -L https://setup.vector.dev)" ``` Alternatively, `CSM_MIGRATE` may be left unset to leave the removal of the diff --git a/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md b/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md index 2418624c1e34cc..e09a8e93ee3b86 100644 --- a/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md @@ -12,15 +12,13 @@ badges: Vector's 0.35.0 release includes **breaking changes**: 1. [The Throttle transform's `events_discarded_total` internal metric is now opt-in](#events-discarded-total-opt-in) - -and **deprecations**: - -1. [Deprecation of `file` internal metric tag for file-based components](#deprecate-file-tag) +1. [The `file` internal metric tag is now opt-in for file-based components](#file-tag-opt-in) and **potentially impactful changes**: 1. [Update `component_sent_bytes_total` to correctly report uncompressed bytes for all sinks](#component-sent-bytes-total) 1. [Update `component_received_bytes_total` to correctly report decompressed bytes for all sources](#component-received-bytes-total) +1. [Update default values for the `request.retry_max_duration_secs` and `request.rate_limit_num` sink configuration options](#request-config-options) We cover them below to help you upgrade quickly: @@ -37,15 +35,12 @@ potentially unbounded cardinality. To view events discarded without the `key` tag, use the `component_discarded_events_total` internal metric. -### Deprecations - -#### Deprecation of `file` internal metric tag for file-based components {#deprecate-file-tag} +#### The `file` internal metric tag is now opt-in for file-based components {#file-tag-opt-in} File-based components (file source, Kubernetes logs source, file sink) now include a `internal_metrics.include_file_tag` config option that determines whether the `file` tag is included on the -component's corresponding internal metrics. This config option defaults to `true` for now to retain the -existing behavior. In the next release, the config option will be updated to default to `false`, as this -`tag` is likely to be of high cardinality. +component's corresponding internal metrics. This config option defaults to `false`, as this `tag` is likely to +be of high cardinality. ### Potentially impactful changes @@ -58,3 +53,20 @@ report uncompressed bytes, rather than compressed bytes, for the `component_sent The Heroku Logs, HTTP Server, Prometheus Remote Write, and Splunk HEC sources now correctly report decompressed bytes, rather than compressed bytes, for the `component_received_bytes_total` internal metric. + +#### Update default values for the `request.retry_max_duration_secs` and `request.rate_limit_num` sink configuration options {#request-config-options} + +The `request.retry_max_duration_secs` config option previously defaulted to `3600` seconds. It now defaults to `30` seconds. + +Also, a bug was fixed that prevented component-level default values from being applied. In particular, this updates the default value +for `request.rate_limit_num` for the following sinks: + +| Sink | Default `request.rate_limit_num` Value | Previous Limit | +|--------------------------------|----------------------------------------|----------------| +| AWS Cloudwatch Metrics Sink | 150 | No limit | +| Azure Blob Storage Sink | 250 | No limit | +| GCP Chronicle Unstructured Sink| 1000 | No limit | +| GCP Cloud Storage Sink | 1000 | No limit | +| GCP Cloud Monitoring Sink | 1000 | No limit | + +Note that all changes described above are reflected in the component reference documentation. diff --git a/website/cue/reference/components/sinks/base/appsignal.cue b/website/cue/reference/components/sinks/base/appsignal.cue index 48b3e4b518f7c5..0046a653a3188f 100644 --- a/website/cue/reference/components/sinks/base/appsignal.cue +++ b/website/cue/reference/components/sinks/base/appsignal.cue @@ -141,7 +141,9 @@ base: components: sinks: appsignal: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -190,6 +192,15 @@ base: components: sinks: appsignal: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -250,12 +261,8 @@ base: components: sinks: appsignal: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -273,11 +280,31 @@ base: components: sinks: appsignal: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index d048ad0d0c98b9..834e43e01175ae 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -527,6 +527,15 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -602,12 +611,8 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -625,11 +630,31 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue index 16e90e50e51933..b487d069d375ab 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue @@ -225,7 +225,9 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -274,6 +276,15 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -329,17 +340,13 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { description: "The maximum number of requests allowed within the `rate_limit_duration_secs` time window." required: false type: uint: { - default: 9223372036854775807 + default: 150 unit: "requests" } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -357,11 +364,31 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index c5ba35bba8deab..b03ddca27fbe39 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -449,7 +449,9 @@ base: components: sinks: aws_kinesis_firehose: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -498,6 +500,15 @@ base: components: sinks: aws_kinesis_firehose: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -558,12 +569,8 @@ base: components: sinks: aws_kinesis_firehose: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -581,11 +588,31 @@ base: components: sinks: aws_kinesis_firehose: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index 1e4e3387b25811..ba28c2931a0993 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -458,7 +458,9 @@ base: components: sinks: aws_kinesis_streams: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -507,6 +509,15 @@ base: components: sinks: aws_kinesis_streams: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -567,12 +578,8 @@ base: components: sinks: aws_kinesis_streams: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -590,11 +597,31 @@ base: components: sinks: aws_kinesis_streams: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index 55ad8f643d934f..762bb5339669ac 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -695,7 +695,9 @@ base: components: sinks: aws_s3: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -744,6 +746,15 @@ base: components: sinks: aws_s3: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -804,12 +815,8 @@ base: components: sinks: aws_s3: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -827,11 +834,31 @@ base: components: sinks: aws_s3: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } @@ -925,6 +952,17 @@ base: components: sinks: aws_s3: configuration: { } } } + timezone: { + description: """ + Timezone to use for any date specifiers in template strings. + + This can refer to any valid timezone as defined in the [TZ database][tzdb], or "local" which refers to the system local timezone. It will default to the [globally configured timezone](https://vector.dev/docs/reference/configuration/global-options/#timezone). + + [tzdb]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + """ + required: false + type: string: examples: ["local", "America/New_York", "EST5EDT"] + } tls: { description: "TLS configuration." required: false diff --git a/website/cue/reference/components/sinks/base/aws_sns.cue b/website/cue/reference/components/sinks/base/aws_sns.cue index 81b67a6c45565a..e582659904dd4a 100644 --- a/website/cue/reference/components/sinks/base/aws_sns.cue +++ b/website/cue/reference/components/sinks/base/aws_sns.cue @@ -401,7 +401,9 @@ base: components: sinks: aws_sns: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -450,6 +452,15 @@ base: components: sinks: aws_sns: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -510,12 +521,8 @@ base: components: sinks: aws_sns: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -533,11 +540,31 @@ base: components: sinks: aws_sns: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/aws_sqs.cue b/website/cue/reference/components/sinks/base/aws_sqs.cue index 28e012cc9a1a56..06eea12139980d 100644 --- a/website/cue/reference/components/sinks/base/aws_sqs.cue +++ b/website/cue/reference/components/sinks/base/aws_sqs.cue @@ -406,7 +406,9 @@ base: components: sinks: aws_sqs: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -455,6 +457,15 @@ base: components: sinks: aws_sqs: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -515,12 +526,8 @@ base: components: sinks: aws_sqs: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -538,11 +545,31 @@ base: components: sinks: aws_sqs: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/axiom.cue b/website/cue/reference/components/sinks/base/axiom.cue index 2de88265fc8f8f..910ef98a29a399 100644 --- a/website/cue/reference/components/sinks/base/axiom.cue +++ b/website/cue/reference/components/sinks/base/axiom.cue @@ -124,6 +124,15 @@ base: components: sinks: axiom: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -199,12 +208,8 @@ base: components: sinks: axiom: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -222,11 +227,31 @@ base: components: sinks: axiom: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/azure_blob.cue b/website/cue/reference/components/sinks/base/azure_blob.cue index 482f9274326130..10777b6a1cc0ea 100644 --- a/website/cue/reference/components/sinks/base/azure_blob.cue +++ b/website/cue/reference/components/sinks/base/azure_blob.cue @@ -443,7 +443,9 @@ base: components: sinks: azure_blob: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -492,6 +494,15 @@ base: components: sinks: azure_blob: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -547,17 +558,13 @@ base: components: sinks: azure_blob: configuration: { description: "The maximum number of requests allowed within the `rate_limit_duration_secs` time window." required: false type: uint: { - default: 9223372036854775807 + default: 250 unit: "requests" } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -575,11 +582,31 @@ base: components: sinks: azure_blob: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/azure_monitor_logs.cue b/website/cue/reference/components/sinks/base/azure_monitor_logs.cue index e9f4c16b7e31f7..66564517290cd1 100644 --- a/website/cue/reference/components/sinks/base/azure_monitor_logs.cue +++ b/website/cue/reference/components/sinks/base/azure_monitor_logs.cue @@ -132,7 +132,9 @@ base: components: sinks: azure_monitor_logs: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -181,6 +183,15 @@ base: components: sinks: azure_monitor_logs: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -241,12 +252,8 @@ base: components: sinks: azure_monitor_logs: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -264,11 +271,31 @@ base: components: sinks: azure_monitor_logs: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/clickhouse.cue b/website/cue/reference/components/sinks/base/clickhouse.cue index 52c00efdd4b719..bc4617a4eeeb8d 100644 --- a/website/cue/reference/components/sinks/base/clickhouse.cue +++ b/website/cue/reference/components/sinks/base/clickhouse.cue @@ -190,7 +190,9 @@ base: components: sinks: clickhouse: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -239,6 +241,15 @@ base: components: sinks: clickhouse: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -299,12 +310,8 @@ base: components: sinks: clickhouse: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -322,11 +329,31 @@ base: components: sinks: clickhouse: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/databend.cue b/website/cue/reference/components/sinks/base/databend.cue index 8bc2434cbb1b78..8302802b59f18c 100644 --- a/website/cue/reference/components/sinks/base/databend.cue +++ b/website/cue/reference/components/sinks/base/databend.cue @@ -288,7 +288,9 @@ base: components: sinks: databend: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -337,6 +339,15 @@ base: components: sinks: databend: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -397,12 +408,8 @@ base: components: sinks: databend: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -420,11 +427,31 @@ base: components: sinks: databend: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/datadog_events.cue b/website/cue/reference/components/sinks/base/datadog_events.cue index e6ca4d22c11cf3..2f109aee91a824 100644 --- a/website/cue/reference/components/sinks/base/datadog_events.cue +++ b/website/cue/reference/components/sinks/base/datadog_events.cue @@ -56,7 +56,9 @@ base: components: sinks: datadog_events: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -105,6 +107,15 @@ base: components: sinks: datadog_events: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -165,12 +176,8 @@ base: components: sinks: datadog_events: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -188,11 +195,31 @@ base: components: sinks: datadog_events: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/datadog_logs.cue b/website/cue/reference/components/sinks/base/datadog_logs.cue index af56186188a53d..0d00aa5b6bc047 100644 --- a/website/cue/reference/components/sinks/base/datadog_logs.cue +++ b/website/cue/reference/components/sinks/base/datadog_logs.cue @@ -195,6 +195,15 @@ base: components: sinks: datadog_logs: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -270,12 +279,8 @@ base: components: sinks: datadog_logs: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -293,11 +298,31 @@ base: components: sinks: datadog_logs: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/datadog_metrics.cue b/website/cue/reference/components/sinks/base/datadog_metrics.cue index 77333e8c2d722c..60a3f098a76ddb 100644 --- a/website/cue/reference/components/sinks/base/datadog_metrics.cue +++ b/website/cue/reference/components/sinks/base/datadog_metrics.cue @@ -98,7 +98,9 @@ base: components: sinks: datadog_metrics: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -147,6 +149,15 @@ base: components: sinks: datadog_metrics: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -207,12 +218,8 @@ base: components: sinks: datadog_metrics: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -230,11 +237,31 @@ base: components: sinks: datadog_metrics: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/datadog_traces.cue b/website/cue/reference/components/sinks/base/datadog_traces.cue index 1564e8f720f2d3..a5b3bef5190d37 100644 --- a/website/cue/reference/components/sinks/base/datadog_traces.cue +++ b/website/cue/reference/components/sinks/base/datadog_traces.cue @@ -122,7 +122,9 @@ base: components: sinks: datadog_traces: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -171,6 +173,15 @@ base: components: sinks: datadog_traces: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -231,12 +242,8 @@ base: components: sinks: datadog_traces: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -254,11 +261,31 @@ base: components: sinks: datadog_traces: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/elasticsearch.cue b/website/cue/reference/components/sinks/base/elasticsearch.cue index 1e039927ff1bc8..2ef047689c975a 100644 --- a/website/cue/reference/components/sinks/base/elasticsearch.cue +++ b/website/cue/reference/components/sinks/base/elasticsearch.cue @@ -591,6 +591,15 @@ base: components: sinks: elasticsearch: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -666,12 +675,8 @@ base: components: sinks: elasticsearch: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -689,11 +694,31 @@ base: components: sinks: elasticsearch: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/file.cue b/website/cue/reference/components/sinks/base/file.cue index 00837da33d4d1c..fe8952486a9878 100644 --- a/website/cue/reference/components/sinks/base/file.cue +++ b/website/cue/reference/components/sinks/base/file.cue @@ -333,10 +333,10 @@ base: components: sinks: file: configuration: { Whether or not to include the "file" tag on the component's corresponding internal metrics. This is useful for distinguishing between different files while monitoring. However, the tag's - cardinality is unbounded. This defaults to true for now but will default to false in a future version. + cardinality is unbounded. """ required: false - type: bool: default: true + type: bool: default: false } } path: { @@ -351,4 +351,15 @@ base: components: sinks: file: configuration: { syntax: "template" } } + timezone: { + description: """ + Timezone to use for any date specifiers in template strings. + + This can refer to any valid timezone as defined in the [TZ database][tzdb], or "local" which refers to the system local timezone. It will default to the [globally configured timezone](https://vector.dev/docs/reference/configuration/global-options/#timezone). + + [tzdb]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + """ + required: false + type: string: examples: ["local", "America/New_York", "EST5EDT"] + } } diff --git a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue index 0c5ab5adf1115c..68c3754b2a5e1d 100644 --- a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue +++ b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue @@ -362,7 +362,9 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -411,6 +413,15 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -466,17 +477,13 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { description: "The maximum number of requests allowed within the `rate_limit_duration_secs` time window." required: false type: uint: { - default: 9223372036854775807 + default: 1000 unit: "requests" } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -494,11 +501,31 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue index 8bcf9922c23a08..102866a47f550a 100644 --- a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue +++ b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue @@ -526,7 +526,9 @@ base: components: sinks: gcp_cloud_storage: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -575,6 +577,15 @@ base: components: sinks: gcp_cloud_storage: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -630,17 +641,13 @@ base: components: sinks: gcp_cloud_storage: configuration: { description: "The maximum number of requests allowed within the `rate_limit_duration_secs` time window." required: false type: uint: { - default: 9223372036854775807 + default: 1000 unit: "requests" } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -658,11 +665,31 @@ base: components: sinks: gcp_cloud_storage: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } @@ -701,6 +728,17 @@ base: components: sinks: gcp_cloud_storage: configuration: { """ } } + timezone: { + description: """ + Timezone to use for any date specifiers in template strings. + + This can refer to any valid timezone as defined in the [TZ database][tzdb], or "local" which refers to the system local timezone. It will default to the [globally configured timezone](https://vector.dev/docs/reference/configuration/global-options/#timezone). + + [tzdb]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + """ + required: false + type: string: examples: ["local", "America/New_York", "EST5EDT"] + } tls: { description: "TLS configuration." required: false diff --git a/website/cue/reference/components/sinks/base/gcp_pubsub.cue b/website/cue/reference/components/sinks/base/gcp_pubsub.cue index c2b12e02401703..0eb77ad77cea0c 100644 --- a/website/cue/reference/components/sinks/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sinks/base/gcp_pubsub.cue @@ -353,7 +353,9 @@ base: components: sinks: gcp_pubsub: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -402,6 +404,15 @@ base: components: sinks: gcp_pubsub: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -462,12 +473,8 @@ base: components: sinks: gcp_pubsub: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -485,11 +492,31 @@ base: components: sinks: gcp_pubsub: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue b/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue index 291261fe478666..ba6457243ab071 100644 --- a/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue +++ b/website/cue/reference/components/sinks/base/gcp_stackdriver_logs.cue @@ -178,7 +178,9 @@ base: components: sinks: gcp_stackdriver_logs: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -227,6 +229,15 @@ base: components: sinks: gcp_stackdriver_logs: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -282,17 +293,13 @@ base: components: sinks: gcp_stackdriver_logs: configuration: { description: "The maximum number of requests allowed within the `rate_limit_duration_secs` time window." required: false type: uint: { - default: 9223372036854775807 + default: 1000 unit: "requests" } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -310,11 +317,31 @@ base: components: sinks: gcp_stackdriver_logs: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue b/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue index 3c62004943d17e..4d37b29ad7514b 100644 --- a/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue +++ b/website/cue/reference/components/sinks/base/gcp_stackdriver_metrics.cue @@ -116,7 +116,9 @@ base: components: sinks: gcp_stackdriver_metrics: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -165,6 +167,15 @@ base: components: sinks: gcp_stackdriver_metrics: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -220,17 +231,13 @@ base: components: sinks: gcp_stackdriver_metrics: configuration: { description: "The maximum number of requests allowed within the `rate_limit_duration_secs` time window." required: false type: uint: { - default: 9223372036854775807 + default: 1000 unit: "requests" } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -248,11 +255,31 @@ base: components: sinks: gcp_stackdriver_metrics: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/greptimedb.cue b/website/cue/reference/components/sinks/base/greptimedb.cue index ab78c8e8f83694..9d4ce20f8a5309 100644 --- a/website/cue/reference/components/sinks/base/greptimedb.cue +++ b/website/cue/reference/components/sinks/base/greptimedb.cue @@ -104,7 +104,9 @@ base: components: sinks: greptimedb: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -153,6 +155,15 @@ base: components: sinks: greptimedb: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -213,12 +224,8 @@ base: components: sinks: greptimedb: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -236,11 +243,31 @@ base: components: sinks: greptimedb: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/honeycomb.cue b/website/cue/reference/components/sinks/base/honeycomb.cue index 9b762b95e873be..29dbc538e8916e 100644 --- a/website/cue/reference/components/sinks/base/honeycomb.cue +++ b/website/cue/reference/components/sinks/base/honeycomb.cue @@ -101,7 +101,9 @@ base: components: sinks: honeycomb: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -150,6 +152,15 @@ base: components: sinks: honeycomb: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -210,12 +221,8 @@ base: components: sinks: honeycomb: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -233,11 +240,31 @@ base: components: sinks: honeycomb: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/http.cue b/website/cue/reference/components/sinks/base/http.cue index 43a14b957d084b..c272ab03bbd720 100644 --- a/website/cue/reference/components/sinks/base/http.cue +++ b/website/cue/reference/components/sinks/base/http.cue @@ -514,6 +514,15 @@ base: components: sinks: http: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -589,12 +598,8 @@ base: components: sinks: http: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -612,11 +617,31 @@ base: components: sinks: http: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/humio_logs.cue b/website/cue/reference/components/sinks/base/humio_logs.cue index 6529d8294f36ab..ef75fd53b37013 100644 --- a/website/cue/reference/components/sinks/base/humio_logs.cue +++ b/website/cue/reference/components/sinks/base/humio_logs.cue @@ -402,7 +402,9 @@ base: components: sinks: humio_logs: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -451,6 +453,15 @@ base: components: sinks: humio_logs: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -511,12 +522,8 @@ base: components: sinks: humio_logs: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -534,11 +541,31 @@ base: components: sinks: humio_logs: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/humio_metrics.cue b/website/cue/reference/components/sinks/base/humio_metrics.cue index c45509ce09b106..f41836314e6b32 100644 --- a/website/cue/reference/components/sinks/base/humio_metrics.cue +++ b/website/cue/reference/components/sinks/base/humio_metrics.cue @@ -204,7 +204,9 @@ base: components: sinks: humio_metrics: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -253,6 +255,15 @@ base: components: sinks: humio_metrics: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -313,12 +324,8 @@ base: components: sinks: humio_metrics: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -336,11 +343,31 @@ base: components: sinks: humio_metrics: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/influxdb_logs.cue b/website/cue/reference/components/sinks/base/influxdb_logs.cue index 3dafc496d1e02a..7b817c559466c8 100644 --- a/website/cue/reference/components/sinks/base/influxdb_logs.cue +++ b/website/cue/reference/components/sinks/base/influxdb_logs.cue @@ -181,7 +181,9 @@ base: components: sinks: influxdb_logs: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -230,6 +232,15 @@ base: components: sinks: influxdb_logs: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -290,12 +301,8 @@ base: components: sinks: influxdb_logs: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -313,11 +320,31 @@ base: components: sinks: influxdb_logs: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/influxdb_metrics.cue b/website/cue/reference/components/sinks/base/influxdb_metrics.cue index 11a14ac6c6b602..72f09ac1366d3f 100644 --- a/website/cue/reference/components/sinks/base/influxdb_metrics.cue +++ b/website/cue/reference/components/sinks/base/influxdb_metrics.cue @@ -135,7 +135,9 @@ base: components: sinks: influxdb_metrics: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -184,6 +186,15 @@ base: components: sinks: influxdb_metrics: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -244,12 +255,8 @@ base: components: sinks: influxdb_metrics: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -267,11 +274,31 @@ base: components: sinks: influxdb_metrics: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/logdna.cue b/website/cue/reference/components/sinks/base/logdna.cue index 2067a596cbc337..3daaa549655476 100644 --- a/website/cue/reference/components/sinks/base/logdna.cue +++ b/website/cue/reference/components/sinks/base/logdna.cue @@ -144,7 +144,9 @@ base: components: sinks: logdna: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -193,6 +195,15 @@ base: components: sinks: logdna: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -253,12 +264,8 @@ base: components: sinks: logdna: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -276,11 +283,31 @@ base: components: sinks: logdna: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/loki.cue b/website/cue/reference/components/sinks/base/loki.cue index 6b381045937ed1..a51e528d88e1d7 100644 --- a/website/cue/reference/components/sinks/base/loki.cue +++ b/website/cue/reference/components/sinks/base/loki.cue @@ -466,7 +466,9 @@ base: components: sinks: loki: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -515,6 +517,15 @@ base: components: sinks: loki: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -575,12 +586,8 @@ base: components: sinks: loki: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -598,11 +605,31 @@ base: components: sinks: loki: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/mezmo.cue b/website/cue/reference/components/sinks/base/mezmo.cue index f4e0b59357fa66..8bba924aea1bc7 100644 --- a/website/cue/reference/components/sinks/base/mezmo.cue +++ b/website/cue/reference/components/sinks/base/mezmo.cue @@ -144,7 +144,9 @@ base: components: sinks: mezmo: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -193,6 +195,15 @@ base: components: sinks: mezmo: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -253,12 +264,8 @@ base: components: sinks: mezmo: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -276,11 +283,31 @@ base: components: sinks: mezmo: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/nats.cue b/website/cue/reference/components/sinks/base/nats.cue index 836ab584fcc33b..3567dc4c98079e 100644 --- a/website/cue/reference/components/sinks/base/nats.cue +++ b/website/cue/reference/components/sinks/base/nats.cue @@ -364,7 +364,9 @@ base: components: sinks: nats: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -413,6 +415,15 @@ base: components: sinks: nats: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -439,7 +450,7 @@ base: components: sinks: nats: configuration: { required: false type: { string: { - default: "adaptive" + default: "none" enum: { adaptive: """ Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature. @@ -473,12 +484,8 @@ base: components: sinks: nats: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -496,11 +503,31 @@ base: components: sinks: nats: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/new_relic.cue b/website/cue/reference/components/sinks/base/new_relic.cue index 535650bb074d4c..86f75a1201553e 100644 --- a/website/cue/reference/components/sinks/base/new_relic.cue +++ b/website/cue/reference/components/sinks/base/new_relic.cue @@ -155,7 +155,9 @@ base: components: sinks: new_relic: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -204,6 +206,15 @@ base: components: sinks: new_relic: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -264,12 +275,8 @@ base: components: sinks: new_relic: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -287,11 +294,31 @@ base: components: sinks: new_relic: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue index 8858c0f99ed749..2ecd4d38998785 100644 --- a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue +++ b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue @@ -311,7 +311,9 @@ base: components: sinks: prometheus_remote_write: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -360,6 +362,15 @@ base: components: sinks: prometheus_remote_write: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -420,12 +431,8 @@ base: components: sinks: prometheus_remote_write: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -443,11 +450,31 @@ base: components: sinks: prometheus_remote_write: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/redis.cue b/website/cue/reference/components/sinks/base/redis.cue index 7c39ffc442c46d..57d985b8416e12 100644 --- a/website/cue/reference/components/sinks/base/redis.cue +++ b/website/cue/reference/components/sinks/base/redis.cue @@ -357,7 +357,9 @@ base: components: sinks: redis: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -406,6 +408,15 @@ base: components: sinks: redis: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -432,7 +443,7 @@ base: components: sinks: redis: configuration: { required: false type: { string: { - default: "adaptive" + default: "none" enum: { adaptive: """ Concurrency will be managed by Vector's [Adaptive Request Concurrency][arc] feature. @@ -466,12 +477,8 @@ base: components: sinks: redis: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -489,11 +496,31 @@ base: components: sinks: redis: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/sematext_logs.cue b/website/cue/reference/components/sinks/base/sematext_logs.cue index 31ac3952b4bfd6..5b24555d64c7fa 100644 --- a/website/cue/reference/components/sinks/base/sematext_logs.cue +++ b/website/cue/reference/components/sinks/base/sematext_logs.cue @@ -111,7 +111,9 @@ base: components: sinks: sematext_logs: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -160,6 +162,15 @@ base: components: sinks: sematext_logs: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -220,12 +231,8 @@ base: components: sinks: sematext_logs: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -243,11 +250,31 @@ base: components: sinks: sematext_logs: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/sematext_metrics.cue b/website/cue/reference/components/sinks/base/sematext_metrics.cue index 7fd5af992c0dd9..fbee85a150d51a 100644 --- a/website/cue/reference/components/sinks/base/sematext_metrics.cue +++ b/website/cue/reference/components/sinks/base/sematext_metrics.cue @@ -93,7 +93,9 @@ base: components: sinks: sematext_metrics: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -142,6 +144,15 @@ base: components: sinks: sematext_metrics: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -202,12 +213,8 @@ base: components: sinks: sematext_metrics: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -225,11 +232,31 @@ base: components: sinks: sematext_metrics: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue index cfb3fc32268aad..dd2921b1cca0dd 100644 --- a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue +++ b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue @@ -454,7 +454,9 @@ base: components: sinks: splunk_hec_logs: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -503,6 +505,15 @@ base: components: sinks: splunk_hec_logs: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -563,12 +574,8 @@ base: components: sinks: splunk_hec_logs: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -586,11 +593,31 @@ base: components: sinks: splunk_hec_logs: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue b/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue index 820454b8e867d4..111c9ba0881590 100644 --- a/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue +++ b/website/cue/reference/components/sinks/base/splunk_hec_metrics.cue @@ -178,7 +178,9 @@ base: components: sinks: splunk_hec_metrics: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -227,6 +229,15 @@ base: components: sinks: splunk_hec_metrics: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -287,12 +298,8 @@ base: components: sinks: splunk_hec_metrics: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -310,11 +317,31 @@ base: components: sinks: splunk_hec_metrics: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sinks/base/vector.cue b/website/cue/reference/components/sinks/base/vector.cue index dcbc25335bae00..49e4f9dcf89349 100644 --- a/website/cue/reference/components/sinks/base/vector.cue +++ b/website/cue/reference/components/sinks/base/vector.cue @@ -85,7 +85,9 @@ base: components: sinks: vector: configuration: { description: """ Middleware settings for outbound requests. - Various settings can be configured, such as concurrency and rate limits, timeouts, etc. + Various settings can be configured, such as concurrency and rate limits, timeouts, retry behavior, etc. + + Note that the retry backoff policy follows the Fibonacci sequence. """ required: false type: object: options: { @@ -134,6 +136,15 @@ base: components: sinks: vector: configuration: { required: false type: uint: default: 1 } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit will not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } rtt_deviation_scale: { description: """ Scale of RTT deviations which are not considered anomalous. @@ -194,12 +205,8 @@ base: components: sinks: vector: configuration: { } } retry_attempts: { - description: """ - The maximum number of retries to make for failed requests. - - The default, for all intents and purposes, represents an infinite number of retries. - """ - required: false + description: "The maximum number of retries to make for failed requests." + required: false type: uint: { default: 9223372036854775807 unit: "retries" @@ -217,11 +224,31 @@ base: components: sinks: vector: configuration: { unit: "seconds" } } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } retry_max_duration_secs: { description: "The maximum amount of time to wait between retries." required: false type: uint: { - default: 3600 + default: 30 unit: "seconds" } } diff --git a/website/cue/reference/components/sources/base/datadog_agent.cue b/website/cue/reference/components/sources/base/datadog_agent.cue index a0dde3557be12c..925f06be26ef6c 100644 --- a/website/cue/reference/components/sources/base/datadog_agent.cue +++ b/website/cue/reference/components/sources/base/datadog_agent.cue @@ -174,12 +174,12 @@ base: components: sources: datadog_agent: configuration: { type: bool: default: false } disable_metrics: { - description: "If this is set to `true`, metrics are not accepted by the component." + description: "If this is set to `true`, metrics (beta) are not accepted by the component." required: false type: bool: default: false } disable_traces: { - description: "If this is set to `true`, traces are not accepted by the component." + description: "If this is set to `true`, traces (alpha) are not accepted by the component." required: false type: bool: default: false } @@ -276,9 +276,9 @@ base: components: sources: datadog_agent: configuration: { } multiple_outputs: { description: """ - If this is set to `true` logs, metrics, and traces are sent to different outputs. + If this is set to `true`, logs, metrics (beta), and traces (alpha) are sent to different outputs. - For a source component named `agent`, the received logs, metrics, and traces can then be + For a source component named `agent`, the received logs, metrics (beta), and traces (alpha) can then be configured as input to other components by specifying `agent.logs`, `agent.metrics`, and `agent.traces`, respectively. """ diff --git a/website/cue/reference/components/sources/base/file.cue b/website/cue/reference/components/sources/base/file.cue index 6f105b7449fdd1..b67a878d22ffc0 100644 --- a/website/cue/reference/components/sources/base/file.cue +++ b/website/cue/reference/components/sources/base/file.cue @@ -213,10 +213,10 @@ base: components: sources: file: configuration: { Whether or not to include the "file" tag on the component's corresponding internal metrics. This is useful for distinguishing between different files while monitoring. However, the tag's - cardinality is unbounded. This defaults to true for now but will default to false in a future version. + cardinality is unbounded. """ required: false - type: bool: default: true + type: bool: default: false } } line_delimiter: { diff --git a/website/cue/reference/components/sources/base/kafka.cue b/website/cue/reference/components/sources/base/kafka.cue index 2f84df38358934..c56036c81fadcc 100644 --- a/website/cue/reference/components/sources/base/kafka.cue +++ b/website/cue/reference/components/sources/base/kafka.cue @@ -361,7 +361,7 @@ base: components: sources: kafka: configuration: { } } metrics: { - description: "Metrics configuration." + description: "Metrics (beta) configuration." required: false type: object: options: topic_lag_metric: { description: "Expose topic lag metrics for all topics and partitions. Metric names are `kafka_consumer_lag`." diff --git a/website/cue/reference/components/sources/base/kubernetes_logs.cue b/website/cue/reference/components/sources/base/kubernetes_logs.cue index 4b839e1ddc3585..34b2516edf636c 100644 --- a/website/cue/reference/components/sources/base/kubernetes_logs.cue +++ b/website/cue/reference/components/sources/base/kubernetes_logs.cue @@ -148,10 +148,10 @@ base: components: sources: kubernetes_logs: configuration: { Whether or not to include the "file" tag on the component's corresponding internal metrics. This is useful for distinguishing between different files while monitoring. However, the tag's - cardinality is unbounded. This defaults to true for now but will default to false in a future version. + cardinality is unbounded. """ required: false - type: bool: default: true + type: bool: default: false } } kube_config_file: { diff --git a/website/cue/reference/components/transforms/base/aggregate.cue b/website/cue/reference/components/transforms/base/aggregate.cue index 844e34df84f153..8f3d8519cc9975 100644 --- a/website/cue/reference/components/transforms/base/aggregate.cue +++ b/website/cue/reference/components/transforms/base/aggregate.cue @@ -4,7 +4,7 @@ base: components: transforms: aggregate: configuration: interval_ms: { description: """ The interval between flushes, in milliseconds. - During this time frame, metrics with the same series data (name, namespace, tags, and so on) are aggregated. + During this time frame, metrics (beta) with the same series data (name, namespace, tags, and so on) are aggregated. """ required: false type: uint: default: 10000 diff --git a/website/cue/reference/components/transforms/base/log_to_metric.cue b/website/cue/reference/components/transforms/base/log_to_metric.cue index 976a3cb07c4a5e..dc822817249e31 100644 --- a/website/cue/reference/components/transforms/base/log_to_metric.cue +++ b/website/cue/reference/components/transforms/base/log_to_metric.cue @@ -1,74 +1,108 @@ package metadata -base: components: transforms: log_to_metric: configuration: metrics: { - description: "A list of metrics to generate." - required: true - type: array: items: type: object: options: { - field: { - description: "Name of the field in the event to generate the metric." - required: true - type: string: syntax: "template" - } - increment_by_value: { - description: "Increments the counter by the value in `field`, instead of only by `1`." - relevant_when: "type = \"counter\"" - required: false - type: bool: default: false - } - kind: { - description: """ - Metric kind. +base: components: transforms: log_to_metric: configuration: { + all_metrics: { + description: """ + Setting this flag changes the behavior of this transformation.
+

Notably the `metrics` field will be ignored.

+

All incoming events will be processed and if possible they will be converted to log events. + Otherwise, only items specified in the 'metrics' field will be processed.

+ 
use serde_json::json;
+			let json_event = json!({
+			    "counter": {
+			        "value": 10.0
+			    },
+			    "kind": "incremental",
+			    "name": "test.transform.counter",
+			    "tags": {
+			        "env": "test_env",
+			        "host": "localhost"
+			    }
+			});
+
+ + This is an example JSON representation of a counter with the following properties: - Metrics can be either absolute of incremental. Absolute metrics represent a sort of "last write wins" scenario, - where the latest absolute value seen is meant to be the actual metric value. In contrast, and perhaps intuitively, - incremental metrics are meant to be additive, such that we don't know what total value of the metric is, but we know - that we'll be adding or subtracting the given value from it. + - `counter`: An object with a single property `value` representing the counter value, in this case, `10.0`). + - `kind`: A string indicating the kind of counter, in this case, "incremental". + - `name`: A string representing the name of the counter, here set to "test.transform.counter". + - `tags`: An object containing additional tags such as "env" and "host". - Generally speaking, most metrics storage systems deal with incremental updates. A notable exception is Prometheus, - which deals with, and expects, absolute values from clients. - """ - relevant_when: "type = \"counter\"" - required: false - type: string: { - default: "incremental" - enum: { - absolute: "Absolute metric." - incremental: "Incremental metric." + Objects that can be processed include counter, histogram, gauge, set and summary. + """ + required: false + type: bool: {} + } + metrics: { + description: "A list of metrics to generate." + required: true + type: array: items: type: object: options: { + field: { + description: "Name of the field in the event to generate the metric." + required: true + type: string: syntax: "template" + } + increment_by_value: { + description: "Increments the counter by the value in `field`, instead of only by `1`." + relevant_when: "type = \"counter\"" + required: false + type: bool: default: false + } + kind: { + description: """ + Metric kind. + + Metrics can be either absolute of incremental. Absolute metrics represent a sort of "last write wins" scenario, + where the latest absolute value seen is meant to be the actual metric value. In contrast, and perhaps intuitively, + incremental metrics are meant to be additive, such that we don't know what total value of the metric is, but we know + that we'll be adding or subtracting the given value from it. + + Generally speaking, most metrics storage systems deal with incremental updates. A notable exception is Prometheus, + which deals with, and expects, absolute values from clients. + """ + relevant_when: "type = \"counter\"" + required: false + type: string: { + default: "incremental" + enum: { + absolute: "Absolute metric." + incremental: "Incremental metric." + } } } - } - name: { - description: """ - Overrides the name of the counter. + name: { + description: """ + Overrides the name of the counter. - If not specified, `field` is used as the name of the metric. - """ - required: false - type: string: syntax: "template" - } - namespace: { - description: "Sets the namespace for the metric." - required: false - type: string: syntax: "template" - } - tags: { - description: "Tags to apply to the metric." - required: false - type: object: options: "*": { - description: "A metric tag." - required: true + If not specified, `field` is used as the name of the metric. + """ + required: false type: string: syntax: "template" } - } - type: { - description: "The type of metric to create." - required: true - type: string: enum: { - counter: "A counter." - gauge: "A gauge." - histogram: "A histogram." - set: "A set." - summary: "A summary." + namespace: { + description: "Sets the namespace for the metric." + required: false + type: string: syntax: "template" + } + tags: { + description: "Tags to apply to the metric." + required: false + type: object: options: "*": { + description: "A metric tag." + required: true + type: string: syntax: "template" + } + } + type: { + description: "The type of metric to create." + required: true + type: string: enum: { + counter: "A counter." + gauge: "A gauge." + histogram: "A histogram." + set: "A set." + summary: "A summary." + } } } } diff --git a/website/cue/reference/releases/0.34.0.cue b/website/cue/reference/releases/0.34.0.cue index 58979db8d5998e..429d0a10fc3dbb 100644 --- a/website/cue/reference/releases/0.34.0.cue +++ b/website/cue/reference/releases/0.34.0.cue @@ -30,6 +30,15 @@ releases: "0.34.0": { be decommissioned on February 28th, 2024. """ + known_issues: [ + """ + The Datadog Metrics sink fails to send a large number of requests due to incorrectly + sized batches [#19110](https://github.com/vectordotdev/vector/issues/19110). This will + be fixed in v0.34.1. Until the fix is released, if you are using the Datadog Metrics + sink, we advise remaining on v0.33.1. + """, + ] + changelog: [ { type: "feat"