chore(vector): Build and patch vector (#1323)

* chore(vector): Init patchable

* chore(stackable-devel): Make a special variant for Vector so that a different rust toolchain can be selected

* chore(stackable-devel): Add note about moving the version to
boil-config.toml once renovate can check there (for consistency)

* chore(nix): Add rust and cargo dependencies

Otherwise cargo can't be found

```
error: the 'cargo' binary, normally provided by the 'cargo' component, is not applicable to the '1.89.0-x86_64-unknown-linux-gnu' toolchain
```

* chore(vector): Build from source (based on ubi9-rust-builder)

NOTE: The ubi9-rust-builder could not be used as it contains `ONBUILD`
steps which we need to run after patchable does it's thing. Also it is
specifically designed for operators and their layout (under `rust/` and
using workspaces).

* chore(nix): Remove unused image-tools

* chore(issue_template/vector): Update instructions for version bumps

* fix(vector): Cherry pick unmerged patch from vectordotdev/vector#24028

NOTE: I removed async/await parts from the original patch as that comes after 0.49.0

```sh
pushd $(cargo patchable checkout vector 0.49.0)

git remote add lfrancke https://github.com/lfrancke/vector

git fetch lfrancke

git cherry-pick 3ce729073f23631dd7b5525be640b5fa15af0223
and git cherry-pick --continue
git commit --amend

popd
cargo patchable export vector 0.49.0
```

* chore(vector): Add maintainer label

This seems to be added to other images, so I'm just copying that.

* chore: Update changelog

* Apply suggestions from code review

Co-authored-by: Techassi <sascha.lautenschlaeger@stackable.tech>

* chore(vector): Remove unused upload script

* chore(vector): Remove old comments, add new todo

---------

Co-authored-by: Techassi <sascha.lautenschlaeger@stackable.tech>

Author: NickLarsenNZ

.github/ISSUE_TEMPLATE/update-base-vector.md

@@ -34,8 +34,9 @@ Add/Change/Remove anything that isn't applicable anymore
 ## Update tasks
 
 - [ ] Update `boil-config.toml` to reflect the agreed upon versions in the spreadsheet (including the removal of old versions).
+- [ ] Update the `stackable-devel/boil-config.toml` for the applicable toolchain version for the vector builder.
 - [ ] Update all `boil-config.toml` files which reference vector.
-- [ ] Upload new version (see `vector/upload_new_vector_version.sh`).
+- [ ] Add any patches required for this version (delete patches for removed versions).
 - [ ] Update other dependencies if applicable (eg: inotify_tools, etc).
 - [ ] Check other operators (getting_started / kuttl / supported-versions) for usage of the versions. Add the PR(s) to the list below.
 - [ ] Update the version in demos. Add the PR(s) to the list below.

CHANGELOG.md

@@ -45,7 +45,7 @@ All notable changes to this project will be documented in this file.
 - stackable-base: Bump ubi9 base image ([#1253]).
 - stackable-devel: Bump ubi9 base image and update cargo-auditable to `0.7.0` ([#1253]).
 - stackable-devel: Bump Rust toolchain to `1.89.0` and cargo-auditable to `0.7.1` ([#1319]).
-- vector: Bump to `0.49.0` ([#1258]).
+- vector: Bump to `0.49.0` and build from source (applying patches) ([#1258], [#1323]).
 - airflow: Bump uvicorn dependency to `0.37.0` ([#1264]).
 - trino-cli: Bump to 477 ([#1285]).
 - tools: Bump dependency versions - kubectl to `1.34.1`, yq to `4.47.2`, and jq to `1.8.1` ([#1290]).
@@ -121,6 +121,7 @@ All notable changes to this project will be documented in this file.
 [#1319]: https://github.com/stackabletech/docker-images/pull/1319
 [#1320]: https://github.com/stackabletech/docker-images/pull/1320
 [#1322]: https://github.com/stackabletech/docker-images/pull/1322
+[#1323]: https://github.com/stackabletech/docker-images/pull/1323
 
 ## [25.7.0] - 2025-07-23
 

nix/sources.json

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+# WARNING: This is currently specific to stackable-operators due to the path to
+# the source (under `rust/`).
+
 # Copy over the binary
 cp "$1" /app
 

shared/copy_artifacts.sh

@@ -5,6 +5,11 @@
 }:
 
 pkgs.mkShell {
+  packages = [
+    pkgs.cargo
+    pkgs.rustc
+  ];
+
   buildInputs = [
     # Required by patchable
     pkgs.openssl

shell.nix

@@ -42,8 +42,10 @@ COPY stackable-base/stackable/curlrc /root/.curlrc
 
 # This SHOULD be kept in sync with operator-templating and other tools to reduce build times
 # Find the latest version here: https://doc.rust-lang.org/stable/releases.html
+# TODO (@NickLarsenNZ): Move the version into boil-config.toml once renovate can look there
 # renovate: datasource=github-releases packageName=rust-lang/rust
-ENV RUST_DEFAULT_TOOLCHAIN_VERSION=1.89.0
+ARG RUST_DEFAULT_TOOLCHAIN_VERSION=1.89.0
+ENV RUST_DEFAULT_TOOLCHAIN_VERSION=${RUST_DEFAULT_TOOLCHAIN_VERSION}
 # Find the latest version here: https://crates.io/crates/cargo-cyclonedx
 # renovate: datasource=crate packageName=cargo-cyclonedx
 ENV CARGO_CYCLONEDX_CRATE_VERSION=0.5.7

stackable-devel/Dockerfile

@@ -1 +1,7 @@
 [versions."1.0.0"]
+
+# Used specifically by vector
+[versions."vector-build".build-arguments]
+# Use what upstream vector uses:
+# https://github.com/vectordotdev/vector/blob/v0.49.0/rust-toolchain.toml
+rust-default-toolchain-version = "1.88.0"

stackable-devel/boil-config.toml

@@ -1,6 +1,90 @@
 # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
 # check=error=true
 
+FROM local-image/stackable-devel AS vector-builder
+
+ARG PRODUCT_VERSION
+ARG RELEASE_VERSION
+ARG STACKABLE_USER_UID
+ARG PROTOC_VERSION
+
+RUN <<EOF
+microdnf update
+microdnf install \
+    `# vector docs say we need these (trying automake instead of autotools)` \
+    cmake \
+    automake \
+    `# openssl libs and related packages required by the build` \
+    perl \
+    findutils \
+    openssl-devel \
+    pkg-config \
+    `# tar needed to create the source code snapshot before building the Rust code` \
+    tar \
+    `# needed for rdkafka-sys` \
+    cyrus-sasl-devel
+microdnf clean all
+rm -rf /var/cache/yum
+EOF
+
+# Container Storage Interface is defined using GRPC/Protobuf, our operators that use it (secret-operator/listener-operator) require
+# protoc via Prost (https://github.com/tokio-rs/prost).
+WORKDIR /opt/protoc
+# Prost does not document which version of protoc it expects (https://docs.rs/prost-build/0.12.4/prost_build/), so this should be the latest upstream version
+# (within reason).
+RUN ARCH=$(arch | sed 's/^aarch64$/aarch_64/') \
+  && curl --fail --location --output protoc.zip "https://repo.stackable.tech/repository/packages/protoc/protoc-${PROTOC_VERSION}-linux-${ARCH}.zip" \
+  && unzip protoc.zip \
+  && rm protoc.zip
+ENV PROTOC=/opt/protoc/bin/protoc
+
+WORKDIR /stackable
+
+COPY --chown=${STACKABLE_USER_UID}:0 vector/stackable/patches/patchable.toml /stackable/src/vector/stackable/patches/patchable.toml
+COPY --chown=${STACKABLE_USER_UID}:0 vector/stackable/patches/${PRODUCT_VERSION} /stackable/src/vector/stackable/patches/${PRODUCT_VERSION}
+
+# Build artifacts will be available in /app.
+RUN mkdir /app
+
+# This script is designed for operators, and their source path.
+# So we can't use it. Instead we use a modified version.
+# COPY shared/copy_artifacts.sh /
+COPY vector/copy_artifacts.sh /
+
+RUN <<EOF
+cd "$(/stackable/patchable --images-repo-root=src checkout vector ${PRODUCT_VERSION})"
+
+NEW_VERSION="${PRODUCT_VERSION}-stackable${RELEASE_VERSION}"
+
+# Create snapshot of the source code including custom patches
+tar -czf /stackable/vector-${NEW_VERSION}-src.tar.gz .
+
+. "$HOME/.cargo/env"
+
+# Build vector with default features
+# TODO (@NickLarsenNZ): Consider reducing the feature-set to only what we need in the sidecar.
+cargo auditable --quiet build --release
+
+# Generate SBOMs and copy them to /app (via a script)
+cargo cyclonedx --all --spec-version 1.5 --describe binaries
+
+# -maxdepth 1: The interesting binaries are all directly in ${BUILD_DIR}.
+# -regex filters out tests
+# - exec copies matching files to /app
+find target/release \
+  -regextype egrep \
+  -maxdepth 1 \
+  -executable \
+  -type f \
+  ! -regex ".*\-[a-fA-F0-9]{16,16}$" \
+  -exec /copy_artifacts.sh {} \;
+
+echo "The following files will be copied to the runtime image: $(ls /app)"
+
+# Set correct permissions
+chmod -R g=u /stackable
+EOF
+
 FROM local-image/stackable-base
 
 ARG PRODUCT_VERSION
@@ -9,6 +93,11 @@ ARG INOTIFY_TOOLS
 ARG TARGETARCH
 ARG STACKABLE_USER_UID
 
+LABEL maintainer="Stackable GmbH"
+
+COPY --chown=${STACKABLE_USER_UID}:0 opa/licenses /licenses
+COPY --from=vector-builder --chown=${STACKABLE_USER_UID}:0 /app/* /usr/local/bin/
+
 # Init Jobs/Pods often start a Vector Sidecar Container which collects the logs.
 # As soon as an Init Container is done it'll need to tell the Vector sidecar that it can now also stop
 # This happens by writing a "shutdown file" in a shared volume
@@ -18,10 +107,7 @@ RUN <<EOF
 ARCH="${TARGETARCH/amd64/x86_64}"
 ARCH="${ARCH/arm64/aarch64}"
 rpm --install \
-    "https://repo.stackable.tech/repository/packages/vector/vector-${PRODUCT_VERSION}-${RPM_RELEASE}.${ARCH}.rpm" \
     "https://repo.stackable.tech/repository/packages/inotify-tools/inotify-tools-${INOTIFY_TOOLS}.${ARCH}.rpm"
-mkdir /licenses
-cp /usr/share/licenses/vector-${PRODUCT_VERSION}/LICENSE /licenses/VECTOR_LICENSE
 
 # Create the directory /stackable/vector/var.
 # This directory is set by operator-rs in the parameter `data_dir`

vector/Dockerfile

@@ -1,6 +1,10 @@
 [versions."0.49.0".local-images]
+stackable-devel = "vector-build"
 stackable-base = "1.0.0"
 
 [versions."0.49.0".build-arguments]
+# See .scripts/upload_new_protoc_version.sh
+# Unsure which version is used by vector. They seem to install `buf` in workflows.
+protoc-version = "31.1"
 inotify-tools = "3.22.1.0-1.el9"
 rpm-release = "1"

vector/boil-config.toml

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# NOTE: This is a modified version of shared/copy_artifacts.sh
+
+# Copy over the binary
+cp "$1" /app
+
+# And now try to find a BOM file named like the binary + _bin.cdx.xml and copy it over as well if it exists
+base=$(basename "$1")
+find /src/ -type f -name "${base}_bin.cdx.xml" -exec cp {} /app \;