From c3ae63fb02c905af468bfdd1e28daec85e0331da Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 14 Apr 2022 16:12:38 +0100 Subject: [PATCH 01/11] Use supervisord to supervise Postgres and Caddy --- docker/complement/SynapseWorkers.Dockerfile | 3 +++ docker/complement/conf-workers/caddy.supervisord.conf | 7 +++++++ .../complement/conf-workers/postgres.supervisord.conf | 11 +++++++++++ .../conf-workers/start-complement-synapse-workers.sh | 6 ------ 4 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 docker/complement/conf-workers/caddy.supervisord.conf create mode 100644 docker/complement/conf-workers/postgres.supervisord.conf diff --git a/docker/complement/SynapseWorkers.Dockerfile b/docker/complement/SynapseWorkers.Dockerfile index 65df2d114d58..7db3400ed50a 100644 --- a/docker/complement/SynapseWorkers.Dockerfile +++ b/docker/complement/SynapseWorkers.Dockerfile @@ -34,6 +34,9 @@ WORKDIR /data # Copy the caddy config COPY conf-workers/caddy.complement.json /root/caddy.json +COPY conf-workers/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf +COPY conf-workers/caddy.supervisord.conf /etc/supervisor/conf.d/caddy.conf + # Copy the entrypoint COPY conf-workers/start-complement-synapse-workers.sh / diff --git a/docker/complement/conf-workers/caddy.supervisord.conf b/docker/complement/conf-workers/caddy.supervisord.conf new file mode 100644 index 000000000000..a8141a5f071f --- /dev/null +++ b/docker/complement/conf-workers/caddy.supervisord.conf @@ -0,0 +1,7 @@ +[program:caddy] +command=/root/caddy run --config /root/caddy.json +autorestart=unexpected +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/docker/complement/conf-workers/postgres.supervisord.conf b/docker/complement/conf-workers/postgres.supervisord.conf new file mode 100644 index 000000000000..0d1ed56071c7 --- /dev/null +++ b/docker/complement/conf-workers/postgres.supervisord.conf @@ -0,0 +1,11 @@ +[program:postgres] +command=/usr/bin/pg_ctlcluster 13 main start --foreground + +# Lower priority number = starts first +priority=100 + +autorestart=unexpected +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/docker/complement/conf-workers/start-complement-synapse-workers.sh b/docker/complement/conf-workers/start-complement-synapse-workers.sh index 2c1e05bd6221..b9a6b55bbe8e 100755 --- a/docker/complement/conf-workers/start-complement-synapse-workers.sh +++ b/docker/complement/conf-workers/start-complement-synapse-workers.sh @@ -12,12 +12,6 @@ function log { # Replace the server name in the caddy config sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json -log "starting postgres" -pg_ctlcluster 13 main start - -log "starting caddy" -/root/caddy start --config /root/caddy.json - # Set the server name of the homeserver export SYNAPSE_SERVER_NAME=${SERVER_NAME} From 93d3d11f98a53338911b73575cedc6a6991ed69e Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 14 Apr 2022 16:16:26 +0100 Subject: [PATCH 02/11] Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) --- changelog.d/12480.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/12480.misc diff --git a/changelog.d/12480.misc b/changelog.d/12480.misc new file mode 100644 index 000000000000..dd608bf9f7b1 --- /dev/null +++ b/changelog.d/12480.misc @@ -0,0 +1 @@ +Use supervisord to supervise Postgres and Caddy in the Complement image. \ No newline at end of file From 19bda0507b3f165467cf94b36abd41333a3a4404 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 21 Apr 2022 10:34:46 +0100 Subject: [PATCH 03/11] Prefix log lines with program names --- docker/complement/conf-workers/caddy.supervisord.conf | 2 +- docker/complement/conf-workers/postgres.supervisord.conf | 2 +- docker/conf-workers/prefix-log | 5 +++++ docker/configure_workers_and_start.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) create mode 100755 docker/conf-workers/prefix-log diff --git a/docker/complement/conf-workers/caddy.supervisord.conf b/docker/complement/conf-workers/caddy.supervisord.conf index a8141a5f071f..d9ddb51dac46 100644 --- a/docker/complement/conf-workers/caddy.supervisord.conf +++ b/docker/complement/conf-workers/caddy.supervisord.conf @@ -1,5 +1,5 @@ [program:caddy] -command=/root/caddy run --config /root/caddy.json +command=/usr/local/bin/prefix-log /root/caddy run --config /root/caddy.json autorestart=unexpected stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 diff --git a/docker/complement/conf-workers/postgres.supervisord.conf b/docker/complement/conf-workers/postgres.supervisord.conf index 0d1ed56071c7..4e061cb72f0e 100644 --- a/docker/complement/conf-workers/postgres.supervisord.conf +++ b/docker/complement/conf-workers/postgres.supervisord.conf @@ -1,5 +1,5 @@ [program:postgres] -command=/usr/bin/pg_ctlcluster 13 main start --foreground +command=/usr/local/bin/prefix-log /usr/bin/pg_ctlcluster 13 main start --foreground # Lower priority number = starts first priority=100 diff --git a/docker/conf-workers/prefix-log b/docker/conf-workers/prefix-log new file mode 100755 index 000000000000..ebd64da5ecde --- /dev/null +++ b/docker/conf-workers/prefix-log @@ -0,0 +1,5 @@ +#!/bin/bash + +exec 1> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&1) +exec 2> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&2) +exec "$@" diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 23cac18e8dbd..70eee362c285 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -171,7 +171,7 @@ # Templates for sections that may be inserted multiple times in config files SUPERVISORD_PROCESS_CONFIG_BLOCK = """ [program:synapse_{name}] -command=/usr/local/bin/python -m {app} \ +command=/usr/local/bin/prefix-log /usr/local/bin/python -m {app} \ --config-path="{config_path}" \ --config-path=/conf/workers/shared.yaml \ --config-path=/conf/workers/{name}.yaml From d0d360d76da4ecba732bdbbbbf59384cd13dca60 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 21 Apr 2022 10:35:00 +0100 Subject: [PATCH 04/11] Remove redundant worker name from log config --- docker/conf/log.config | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docker/conf/log.config b/docker/conf/log.config index 7a216a36a046..615a74e95bc4 100644 --- a/docker/conf/log.config +++ b/docker/conf/log.config @@ -2,11 +2,7 @@ version: 1 formatters: precise: -{% if worker_name %} - format: '%(asctime)s - worker:{{ worker_name }} - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' -{% else %} - format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' -{% endif %} + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' handlers: {% if LOG_FILE_PATH %} From ea23bcd05310f698effd772ab6300899b2a86666 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 21 Apr 2022 10:35:22 +0100 Subject: [PATCH 05/11] Don't use `sh -c` for the entrypoint as it intercepts signals --- docker/complement/SynapseWorkers.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/complement/SynapseWorkers.Dockerfile b/docker/complement/SynapseWorkers.Dockerfile index 7db3400ed50a..9a4438e7303b 100644 --- a/docker/complement/SynapseWorkers.Dockerfile +++ b/docker/complement/SynapseWorkers.Dockerfile @@ -43,7 +43,7 @@ COPY conf-workers/start-complement-synapse-workers.sh / # Expose caddy's listener ports EXPOSE 8008 8448 -ENTRYPOINT /start-complement-synapse-workers.sh +ENTRYPOINT ["/start-complement-synapse-workers.sh"] # Update the healthcheck to have a shorter check interval HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \ From 70b80e3a1f16d979647d9bfda8f827ba0a5f4b64 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 21 Apr 2022 11:59:52 +0100 Subject: [PATCH 06/11] fixup! Remove redundant worker name from log config --- docker/conf/log.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/conf/log.config b/docker/conf/log.config index 615a74e95bc4..dc8c70befd40 100644 --- a/docker/conf/log.config +++ b/docker/conf/log.config @@ -2,7 +2,7 @@ version: 1 formatters: precise: - format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' handlers: {% if LOG_FILE_PATH %} From 9ef24471411d01d55403314c7c5c2f5548a09f4d Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 21 Apr 2022 13:58:18 +0100 Subject: [PATCH 07/11] Fix prefix-log path --- docker/Dockerfile-workers | 3 +++ docker/{conf-workers => }/prefix-log | 0 2 files changed, 3 insertions(+) rename docker/{conf-workers => }/prefix-log (100%) diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index 9ccb2b22a750..24b03585f9a2 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -20,6 +20,9 @@ RUN rm /etc/nginx/sites-enabled/default # Copy Synapse worker, nginx and supervisord configuration template files COPY ./docker/conf-workers/* /conf/ +# Copy a script to prefix log lines with the supervisor program name +COPY ./docker/prefix-log /usr/local/bin/ + # Expose nginx listener port EXPOSE 8080/tcp diff --git a/docker/conf-workers/prefix-log b/docker/prefix-log similarity index 100% rename from docker/conf-workers/prefix-log rename to docker/prefix-log From a65fa98fbb7c373700514588edd674133b84d1fe Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 21 Apr 2022 13:58:26 +0100 Subject: [PATCH 08/11] Use fast shutdown mode for Postgres --- docker/complement/conf-workers/postgres.supervisord.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/complement/conf-workers/postgres.supervisord.conf b/docker/complement/conf-workers/postgres.supervisord.conf index 4e061cb72f0e..c9edb6fbb9bb 100644 --- a/docker/complement/conf-workers/postgres.supervisord.conf +++ b/docker/complement/conf-workers/postgres.supervisord.conf @@ -9,3 +9,8 @@ stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 + +# Use 'Fast Shutdown' mode which aborts current transactions and closes connections quickly. +# (Default (TERM) is 'Smart Shutdown' which stops accepting new connections but +# lets existing connections close gracefully.) +stopsignal=INT From 0df4a91496a6640e5b7709d390df3f1e35cf9b0e Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 27 Apr 2022 13:59:18 +0100 Subject: [PATCH 09/11] Update changelog.d/12480.misc Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- changelog.d/12480.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/12480.misc b/changelog.d/12480.misc index dd608bf9f7b1..18a85e7b1579 100644 --- a/changelog.d/12480.misc +++ b/changelog.d/12480.misc @@ -1 +1 @@ -Use supervisord to supervise Postgres and Caddy in the Complement image. \ No newline at end of file +Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time. \ No newline at end of file From 32ae5e1d3254ac154d6ad8c5e8c1192ebff3b9e7 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 27 Apr 2022 14:01:24 +0100 Subject: [PATCH 10/11] Add a comment describing prefix-log --- docker/prefix-log | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/prefix-log b/docker/prefix-log index ebd64da5ecde..0e26a4f19d33 100755 --- a/docker/prefix-log +++ b/docker/prefix-log @@ -1,4 +1,11 @@ #!/bin/bash +# +# Prefixes all lines on stdout and stderr with the process name (as determined by +# the SUPERVISOR_PROCESS_NAME env var, which is automatically set by Supervisor). +# +# Usage: +# prefix-log command [args...] +# exec 1> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&1) exec 2> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&2) From 014032724772c08a35da57011afc8d045261e77f Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 27 Apr 2022 14:01:50 +0100 Subject: [PATCH 11/11] Give Postgres priority 1 to match Redis --- docker/complement/conf-workers/postgres.supervisord.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/complement/conf-workers/postgres.supervisord.conf b/docker/complement/conf-workers/postgres.supervisord.conf index c9edb6fbb9bb..5608342d1a9e 100644 --- a/docker/complement/conf-workers/postgres.supervisord.conf +++ b/docker/complement/conf-workers/postgres.supervisord.conf @@ -2,7 +2,7 @@ command=/usr/local/bin/prefix-log /usr/bin/pg_ctlcluster 13 main start --foreground # Lower priority number = starts first -priority=100 +priority=1 autorestart=unexpected stdout_logfile=/dev/stdout