From e01ede79c3b8a2910a244e837d7fd089bdc28f7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Sat, 27 Apr 2024 20:32:04 +0200 Subject: [PATCH 1/2] syslog-ng: add webhook() and webhook-json() sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- .github/workflows/syslog-ng-docker.yml | 9 ++ .github/workflows/syslog-ng-snapshot.yml | 33 ++++- .github/workflows/syslog-ng-stable.yml | 22 +++ syslog-ng/alpine.dockerfile | 4 +- syslog-ng/apkbuild/axoflow/syslog-ng/APKBUILD | 8 +- syslog-ng/python-modules/requirements.txt | 1 + syslog-ng/python-modules/webhook/__init__.py | 5 + .../python-modules/webhook/scl/webhook.conf | 65 ++++++++ syslog-ng/python-modules/webhook/source.py | 139 ++++++++++++++++++ 9 files changed, 280 insertions(+), 6 deletions(-) create mode 100644 syslog-ng/python-modules/requirements.txt create mode 100644 syslog-ng/python-modules/webhook/__init__.py create mode 100644 syslog-ng/python-modules/webhook/scl/webhook.conf create mode 100644 syslog-ng/python-modules/webhook/source.py diff --git a/.github/workflows/syslog-ng-docker.yml b/.github/workflows/syslog-ng-docker.yml index f2bfe32a0c..6f9c597241 100644 --- a/.github/workflows/syslog-ng-docker.yml +++ b/.github/workflows/syslog-ng-docker.yml @@ -9,6 +9,9 @@ on: tarball-artifact: required: false type: string + axosyslog-modules-artifact: + required: true + type: string snapshot-version: required: false type: string @@ -34,6 +37,12 @@ jobs: name: ${{ inputs.tarball-artifact }} path: syslog-ng/apkbuild/axoflow/syslog-ng + - name: Download axosyslog-modules tarball artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.axosyslog-modules-artifact }} + path: syslog-ng/apkbuild/axoflow/syslog-ng + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx diff --git a/.github/workflows/syslog-ng-snapshot.yml b/.github/workflows/syslog-ng-snapshot.yml index bf471bdd18..880cd8218f 100644 --- a/.github/workflows/syslog-ng-snapshot.yml +++ b/.github/workflows/syslog-ng-snapshot.yml @@ -59,9 +59,29 @@ jobs: name: source-tarball path: dbld/build/*.tar.* + axosyslog-modules: + name: axosyslog-modules + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Create AxoSyslog modules tarball + working-directory: syslog-ng + run: | + tar -czvf python-modules.tar.gz python-modules/ + + - name: Store axosyslog-modules tarball as artifact + uses: actions/upload-artifact@v4 + with: + name: axosyslog-modules-tarball + path: syslog-ng/python-modules.tar.gz + build-and-test: runs-on: ubuntu-latest - needs: tarball + needs: + - tarball + - axosyslog-modules steps: - name: Checkout source uses: actions/checkout@v4 @@ -72,7 +92,13 @@ jobs: name: source-tarball path: syslog-ng/apkbuild/axoflow/syslog-ng - - name: Build and export Docker image + - name: Download axosyslog-modules tarball artifact + uses: actions/download-artifact@v4 + with: + name: axosyslog-modules-tarball + path: syslog-ng/apkbuild/axoflow/syslog-ng + + - name: Build Docker image uses: docker/build-push-action@v5 with: context: syslog-ng @@ -93,10 +119,11 @@ jobs: publish-image: if: github.ref == 'refs/heads/main' uses: ./.github/workflows/syslog-ng-docker.yml - needs: [tarball, build-and-test] + needs: [tarball, build-and-test, axosyslog-modules] with: pkg-type: nightly tarball-artifact: source-tarball + axosyslog-modules-artifact: axosyslog-modules-tarball snapshot-version: ${{ needs.tarball.outputs.snapshot-version }} # https://github.com/actions/delete-package-versions/issues/90 diff --git a/.github/workflows/syslog-ng-stable.yml b/.github/workflows/syslog-ng-stable.yml index 5fd358a268..ffd090b34b 100644 --- a/.github/workflows/syslog-ng-stable.yml +++ b/.github/workflows/syslog-ng-stable.yml @@ -23,6 +23,24 @@ jobs: outputs: version: ${{ steps.unpack_tag.outputs.group1 }} + axosyslog-modules: + name: axosyslog-modules + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Create AxoSyslog modules tarball + working-directory: syslog-ng + run: | + tar -czvf python-modules.tar.gz python-modules/ + + - name: Store axosyslog-modules tarball as artifact + uses: actions/upload-artifact@v4 + with: + name: axosyslog-modules-tarball + path: syslog-ng/python-modules.tar.gz + publish-packages: uses: ./.github/workflows/syslog-ng-packages.yml needs: @@ -33,5 +51,9 @@ jobs: publish-image: uses: ./.github/workflows/syslog-ng-docker.yml + needs: + - prepare + - axosyslog-modules with: pkg-type: stable + axosyslog-modules-artifact: axosyslog-modules-tarball diff --git a/syslog-ng/alpine.dockerfile b/syslog-ng/alpine.dockerfile index f4ee7e015f..dddede67b3 100644 --- a/syslog-ng/alpine.dockerfile +++ b/syslog-ng/alpine.dockerfile @@ -26,11 +26,11 @@ RUN mkdir packages \ && printf 'export JOBS=$(nproc)\nexport MAKEFLAGS=-j$JOBS\n' >> .abuild/abuild.conf \ && cd axoflow/syslog-ng \ && if [ "$PKG_TYPE" = "nightly" ]; then \ - tarball_filename="$(ls *.tar.*)"; \ + tarball_filename="$(ls syslog-ng-*.tar.*)"; \ [ -z "$tarball_filename" ] && echo "Tarball for nightly can not be found" && exit 1; \ tarball_name="${tarball_filename/\.tar.*}"; \ sed -i -e "s|^pkgver=.*|pkgver=$SNAPSHOT_VERSION|" -e "s|^builddir=.*|builddir=\"\$srcdir/$tarball_name\"|" APKBUILD; \ - sed -i -e "s|^source=.*|source=\"$tarball_filename\"|" APKBUILD; \ + sed -i -e "s|^source=.*|source=\"$tarball_filename|" APKBUILD; \ fi \ && abuild checksum \ && abuild -r diff --git a/syslog-ng/apkbuild/axoflow/syslog-ng/APKBUILD b/syslog-ng/apkbuild/axoflow/syslog-ng/APKBUILD index e0c62c5f9a..3e7618ae1b 100644 --- a/syslog-ng/apkbuild/axoflow/syslog-ng/APKBUILD +++ b/syslog-ng/apkbuild/axoflow/syslog-ng/APKBUILD @@ -51,7 +51,9 @@ subpackages=" $pkgname-python3:_python3 $pkgname-grpc:_grpc " -source="https://github.com/syslog-ng/syslog-ng/releases/download/syslog-ng-$pkgver/syslog-ng-$pkgver.tar.gz" +source="https://github.com/syslog-ng/syslog-ng/releases/download/syslog-ng-$pkgver/syslog-ng-$pkgver.tar.gz + python-modules.tar.gz + " builddir="$srcdir/$pkgname-$pkgver" _modules=" @@ -145,6 +147,9 @@ _python3() { depends="$pkgname=$pkgver-r$pkgrel python3" install="$subpkgname.post-install" + cat $srcdir/python-modules/requirements.txt >> $pkgdir/usr/lib/syslog-ng/python/requirements.txt + rm $srcdir/python-modules/requirements.txt + mv $srcdir/python-modules/* $pkgdir/usr/lib/syslog-ng/python/syslogng/modules _submv usr/lib/syslog-ng/libmod-python.so usr/lib/syslog-ng/python usr/bin/syslog-ng-update-virtualenv } @@ -174,4 +179,5 @@ _submv() { sha512sums=" 2f1e0dea4c0ecfc3c77df7e6ac231ee8436c9c78fcb4df8ccdc417fea7d56791fdeb0844ac35f0342ce7c2bea5618d8723b6b54319c556120099eb809873082e syslog-ng-4.7.1.tar.gz +SKIP python-modules.tar.gz " diff --git a/syslog-ng/python-modules/requirements.txt b/syslog-ng/python-modules/requirements.txt new file mode 100644 index 0000000000..2295c8f4a3 --- /dev/null +++ b/syslog-ng/python-modules/requirements.txt @@ -0,0 +1 @@ +tornado==6.4 diff --git a/syslog-ng/python-modules/webhook/__init__.py b/syslog-ng/python-modules/webhook/__init__.py new file mode 100644 index 0000000000..8ccf273a6a --- /dev/null +++ b/syslog-ng/python-modules/webhook/__init__.py @@ -0,0 +1,5 @@ +from .source import HTTPSource + +__all__ = [ + "HTTPSource" +] diff --git a/syslog-ng/python-modules/webhook/scl/webhook.conf b/syslog-ng/python-modules/webhook/scl/webhook.conf new file mode 100644 index 0000000000..a788878bbe --- /dev/null +++ b/syslog-ng/python-modules/webhook/scl/webhook.conf @@ -0,0 +1,65 @@ +############################################################################# +# Copyright (c) 2024 László Várady +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 as published +# by the Free Software Foundation, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# As an additional exemption you are allowed to compile & link against the +# OpenSSL libraries as published by the OpenSSL project. See the file +# COPYING for details. +# +############################################################################# + +block source webhook( + port("") + auth_token("") + tls_key_file("") + tls_cert_file("") + ... +) +{ + python( + class("syslogng.modules.webhook.HTTPSource") + options( + "port" => "`port`" + "auth_token" => "`auth_token`" + "tls_key_file" => "`tls_key_file`" + "tls_cert_file" => "`tls_cert_file`" + ) + `__VARARGS__` + ); +}; + +block source webhook-json( + port("") + auth_token("") + tls_key_file("") + tls_cert_file("") + prefix("") + ... +) +{ + channel { + source { + webhook( + port("`port`") auth_token("`auth_token`") + tls_key_file("`tls_key_file`") tls_cert_file("`tls_cert_file`") + `__VARARGS__` + ); + }; + + parser { + json-parser(prefix("`prefix`")); + }; + }; +}; diff --git a/syslog-ng/python-modules/webhook/source.py b/syslog-ng/python-modules/webhook/source.py new file mode 100644 index 0000000000..66836db101 --- /dev/null +++ b/syslog-ng/python-modules/webhook/source.py @@ -0,0 +1,139 @@ +############################################################################# +# Copyright (c) 2024 László Várady +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 as published +# by the Free Software Foundation, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# As an additional exemption you are allowed to compile & link against the +# OpenSSL libraries as published by the OpenSSL project. See the file +# COPYING for details. +# +############################################################################# + +from syslogng import LogSource, LogMessage + +import logging +import asyncio +import threading +import tornado +import ssl +import signal +from typing import Any + +signal.signal(signal.SIGINT, signal.SIG_IGN) +signal.signal(signal.SIGTERM, signal.SIG_IGN) + +class Handler(tornado.web.RequestHandler): + def initialize(self, source) -> None: + self.source = source + + @tornado.web.authenticated + async def post(self) -> None: + # racy, but the client should retry + if self.source.suspended.is_set(): + self.set_status(503) + await self.finish({"status": "flow-controlled"}) + return + + self.source.post_message(LogMessage(self.request.body)) + await self.finish({"status": "received"}) + + def set_default_headers(self) -> None: + self.set_header("Server", "syslog-ng"); + + def get_current_user(self): + if not self.source.auth_token: + return True + + token = self.request.headers.get("Authorization", "").split(" ") + if len(token) != 2: + return False + + token = token[1] + return self.source.auth_token == token + + def write_error(self, status_code: int, **kwargs: Any) -> None: + self.set_status(status_code) + +class HTTPSource(LogSource): + def init(self, options: dict[str, Any]) -> bool: + self.logger = logging.getLogger("http") + self.set_transport_name("http") + if not self.init_options(options): + return False + + self.ssl_ctx = None + if self.tls_key_file: + self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.ssl_ctx.load_cert_chain(self.tls_cert_file, self.tls_key_file) + + if not self.port: + self.port = 443 if self.tls_key_file else 80 + + self.port = int(self.port) + + self.suspended = threading.Event() + self.event_loop = asyncio.new_event_loop() + self.request_exit = asyncio.Event() + self.app = tornado.web.Application( + [ + (r"/.*", Handler, {"source": self}), + ], + log_function=self.log_access, + autoreload=False, + compress_response=False, + ) + self.server = None + + return True + + async def runServer(self) -> None: + self.logger.info(f"HTTP(S) server started on port {self.port}") + self.server = self.app.listen(self.port, decompress_request=True, ssl_options=self.ssl_ctx) + await self.request_exit.wait() + + self.server.stop() + await self.server.close_all_connections() + + async def stopServer(self) -> None: + self.request_exit.set() + + def deinit(self) -> None: + pass + + def run(self) -> None: + self.event_loop.run_until_complete(self.runServer()) + + def suspend(self) -> None: + self.suspended.set() + + def wakeup(self) -> None: + self.suspended.clear() + + def request_exit(self) -> None : + asyncio.run_coroutine_threadsafe(self.stopServer(), self.event_loop) + pass + + def log_access(self, req: tornado.web.RequestHandler): + self.logger.debug(f"{req.get_status()} {req._request_summary()}") + + def init_options(self, options: dict[str, Any]) -> bool: + try: + self.port = options.get("port") + self.auth_token = options.get("auth_token") + self.tls_key_file = options.get("tls_key_file") + self.tls_cert_file = options.get("tls_cert_file") + return True + except KeyError as e: + self.logger.error(f"Missing option '{e.args[0]}'") + return False From 71d3e9a2424eb0dc63b78f3684a93dbe49629f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Tue, 30 Apr 2024 20:09:50 +0200 Subject: [PATCH 2/2] webhook: implement TLS client authentication MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: László Várady --- .../python-modules/webhook/scl/webhook.conf | 23 ++++++++++- syslog-ng/python-modules/webhook/source.py | 38 +++++++++++++++---- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/syslog-ng/python-modules/webhook/scl/webhook.conf b/syslog-ng/python-modules/webhook/scl/webhook.conf index a788878bbe..ba94383680 100644 --- a/syslog-ng/python-modules/webhook/scl/webhook.conf +++ b/syslog-ng/python-modules/webhook/scl/webhook.conf @@ -25,6 +25,12 @@ block source webhook( auth_token("") tls_key_file("") tls_cert_file("") + + # for client authentication + tls_peer_verify(no) + tls_use_system_cert_store(no) + tls_ca_file("") + tls_ca_dir("") ... ) { @@ -35,6 +41,11 @@ block source webhook( "auth_token" => "`auth_token`" "tls_key_file" => "`tls_key_file`" "tls_cert_file" => "`tls_cert_file`" + + "tls_peer_verify" => `tls_peer_verify` + "tls_use_system_cert_store" => `tls_use_system_cert_store` + "tls_ca_file" => "`tls_ca_file`" + "tls_ca_dir" => "`tls_ca_dir`" ) `__VARARGS__` ); @@ -43,9 +54,15 @@ block source webhook( block source webhook-json( port("") auth_token("") + prefix("") tls_key_file("") tls_cert_file("") - prefix("") + + # for client authentication + tls_peer_verify(no) + tls_use_system_cert_store(no) + tls_ca_file("") + tls_ca_dir("") ... ) { @@ -54,6 +71,10 @@ block source webhook-json( webhook( port("`port`") auth_token("`auth_token`") tls_key_file("`tls_key_file`") tls_cert_file("`tls_cert_file`") + + tls_peer_verify(`tls_peer_verify`) + tls_use_system_cert_store(`tls_use_system_cert_store`) + tls_ca_file("`tls_ca_file`") tls_ca_dir("`tls_ca_dir`") `__VARARGS__` ); }; diff --git a/syslog-ng/python-modules/webhook/source.py b/syslog-ng/python-modules/webhook/source.py index 66836db101..0e8999cce4 100644 --- a/syslog-ng/python-modules/webhook/source.py +++ b/syslog-ng/python-modules/webhook/source.py @@ -57,6 +57,7 @@ def get_current_user(self): token = self.request.headers.get("Authorization", "").split(" ") if len(token) != 2: + self.source.logger.debug("Auth failed, missing Authorization header or auth-scheme") return False token = token[1] @@ -72,16 +73,14 @@ def init(self, options: dict[str, Any]) -> bool: if not self.init_options(options): return False - self.ssl_ctx = None - if self.tls_key_file: - self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - self.ssl_ctx.load_cert_chain(self.tls_cert_file, self.tls_key_file) - if not self.port: self.port = 443 if self.tls_key_file else 80 - self.port = int(self.port) + self.ssl_ctx = None + if self.tls_key_file: + self.setup_tls() + self.suspended = threading.Event() self.event_loop = asyncio.new_event_loop() self.request_exit = asyncio.Event() @@ -124,15 +123,40 @@ def request_exit(self) -> None : asyncio.run_coroutine_threadsafe(self.stopServer(), self.event_loop) pass - def log_access(self, req: tornado.web.RequestHandler): + def log_access(self, req: tornado.web.RequestHandler) -> None: self.logger.debug(f"{req.get_status()} {req._request_summary()}") + def setup_tls(self) -> None: + self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.ssl_ctx.load_cert_chain(certfile=self.tls_cert_file, keyfile=self.tls_key_file) + + if self.tls_peer_verify: + self.logger.debug("Enabling client cert verification") + self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED + else: + self.ssl_ctx.verify_mode = ssl.CERT_NONE + + if self.tls_use_system_cert_store: + self.logger.debug("Using system cert store for client auth") + self.ssl_ctx.load_default_certs(ssl.Purpose.CLIENT_AUTH) + + self.tls_ca_dir = self.tls_ca_dir if self.tls_ca_dir else None + self.tls_ca_file = self.tls_ca_file if self.tls_ca_file else None + + if self.tls_ca_dir or self.tls_ca_file: + self.ssl_ctx.load_verify_locations(cafile=self.tls_ca_file, capath=self.tls_ca_dir) + def init_options(self, options: dict[str, Any]) -> bool: try: self.port = options.get("port") self.auth_token = options.get("auth_token") self.tls_key_file = options.get("tls_key_file") self.tls_cert_file = options.get("tls_cert_file") + + self.tls_peer_verify = bool(options.get("tls_peer_verify", False)) + self.tls_use_system_cert_store = bool(options.get("tls_use_system_cert_store", False)) + self.tls_ca_file = options.get("tls_ca_file") + self.tls_ca_dir = options.get("tls_ca_dir") return True except KeyError as e: self.logger.error(f"Missing option '{e.args[0]}'")