From dfd4235cc827ba8e33e639be521624ef0e071bd9 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Tue, 9 Jul 2024 12:48:18 -0700 Subject: [PATCH 01/10] first pass --- .env | 1 + docker-compose.yml | 207 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 180 insertions(+), 28 deletions(-) diff --git a/.env b/.env index 8e54a3df2c..ccfc7594e6 100644 --- a/.env +++ b/.env @@ -1,4 +1,5 @@ COMPOSE_PROJECT_NAME=sentry-self-hosted +COMPOSE_PROFILES=errors-only SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more diff --git a/docker-compose.yml b/docker-compose.yml index f88d74d99e..f4433bf900 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,11 +23,13 @@ x-sentry-defaults: &sentry_defaults context: ./sentry args: - SENTRY_IMAGE - depends_on: + depends_on: &sentry_depends_on redis: <<: *depends_on-healthy kafka: <<: *depends_on-healthy + zookeeper: + <<: *depends_on-default postgres: <<: *depends_on-healthy memcached: @@ -36,23 +38,19 @@ x-sentry-defaults: &sentry_defaults <<: *depends_on-default snuba-api: <<: *depends_on-default + symbolicator: + <<: *depends_on-default snuba-errors-consumer: <<: *depends_on-default snuba-outcomes-consumer: <<: *depends_on-default snuba-outcomes-billing-consumer: <<: *depends_on-default - snuba-transactions-consumer: - <<: *depends_on-default snuba-subscription-consumer-events: <<: *depends_on-default - snuba-subscription-consumer-transactions: - <<: *depends_on-default snuba-replacer: <<: *depends_on-default - symbolicator: - <<: *depends_on-default - vroom: + snuba-group-attributes-consumer: <<: *depends_on-default entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] @@ -80,6 +78,36 @@ x-sentry-defaults: &sentry_defaults - "./sentry:/etc/sentry" - "./geoip:/geoip:ro" - "./certificates:/usr/local/share/ca-certificates:ro" +x-sentry-defaults-feature-complete: &sentry_defaults-feature-complete + <<: *sentry_defaults + depends_on: + <<: *sentry_depends_on + snuba-transactions-consumer: + <<: *depends_on-default + snuba-replays-consumer: + <<: *depends_on-default + snuba-issue-occurrence-consumer: + <<: *depends_on-default + snuba-metrics-consumer: + <<: *depends_on-default + snuba-generic-metrics-distributions-consumer: + <<: *depends_on-default + snuba-generic-metrics-sets-consumer: + <<: *depends_on-default + snuba-generic-metrics-counters-consumer: + <<: *depends_on-default + snuba-subscription-consumer-transactions: + <<: *depends_on-default + snuba-subscription-consumer-metrics: + <<: *depends_on-default + snuba-profiling-profiles-consumer: + <<: *depends_on-default + snuba-profiling-functions-consumer: + <<: *depends_on-default + snuba-spans-consumer: + <<: *depends_on-default + vroom: + <<: *depends_on-default x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: @@ -268,52 +296,79 @@ services: snuba-outcomes-billing-consumer: <<: *snuba_defaults command: rust-consumer --storage outcomes_raw --consumer-group snuba-consumers --auto-offset-reset=earliest --max-batch-time-ms 750 --no-strict-offset-reset --raw-events-topic outcomes-billing + snuba-group-attributes-consumer: + <<: *snuba_defaults + command: rust-consumer --storage group_attributes --consumer-group snuba-group-attributes-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + snuba-replacer: + <<: *snuba_defaults + command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset + snuba-subscription-consumer-events: + <<: *snuba_defaults + command: subscriptions-scheduler-executor --dataset events --entity events --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-events-subscriptions-consumers --followed-consumer-group=snuba-consumers --schedule-ttl=60 --stale-threshold-seconds=900 + ############################################# + ## Feature Complete Sentry Snuba Consumers ## + ############################################# # Kafka consumer responsible for feeding transactions data into Clickhouse snuba-transactions-consumer: <<: *snuba_defaults command: rust-consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + profiles: + - feature-complete snuba-replays-consumer: <<: *snuba_defaults command: rust-consumer --storage replays --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + profiles: + - feature-complete snuba-issue-occurrence-consumer: <<: *snuba_defaults command: rust-consumer --storage search_issues --consumer-group generic_events_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + profiles: + - feature-complete snuba-metrics-consumer: <<: *snuba_defaults command: rust-consumer --storage metrics_raw --consumer-group snuba-metrics-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset - snuba-group-attributes-consumer: + profiles: + - feature-complete + snuba-subscription-consumer-transactions: <<: *snuba_defaults - command: rust-consumer --storage group_attributes --consumer-group snuba-group-attributes-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --schedule-ttl=60 --stale-threshold-seconds=900 + profiles: + - feature-complete + snuba-subscription-consumer-metrics: + <<: *snuba_defaults + command: subscriptions-scheduler-executor --dataset metrics --entity metrics_sets --entity metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-metrics-subscriptions-consumers --followed-consumer-group=snuba-metrics-consumers --schedule-ttl=60 --stale-threshold-seconds=900 + profiles: + - feature-complete snuba-generic-metrics-distributions-consumer: <<: *snuba_defaults command: rust-consumer --storage generic_metrics_distributions_raw --consumer-group snuba-gen-metrics-distributions-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + profiles: + - feature-complete snuba-generic-metrics-sets-consumer: <<: *snuba_defaults command: rust-consumer --storage generic_metrics_sets_raw --consumer-group snuba-gen-metrics-sets-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + profiles: + - feature-complete snuba-generic-metrics-counters-consumer: <<: *snuba_defaults command: rust-consumer --storage generic_metrics_counters_raw --consumer-group snuba-gen-metrics-counters-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset - snuba-replacer: - <<: *snuba_defaults - command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset - snuba-subscription-consumer-events: - <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset events --entity events --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-events-subscriptions-consumers --followed-consumer-group=snuba-consumers --schedule-ttl=60 --stale-threshold-seconds=900 - snuba-subscription-consumer-transactions: - <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --schedule-ttl=60 --stale-threshold-seconds=900 - snuba-subscription-consumer-metrics: - <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset metrics --entity metrics_sets --entity metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-metrics-subscriptions-consumers --followed-consumer-group=snuba-metrics-consumers --schedule-ttl=60 --stale-threshold-seconds=900 + profiles: + - feature-complete snuba-profiling-profiles-consumer: <<: *snuba_defaults command: rust-consumer --storage profiles --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset + profiles: + - feature-complete snuba-profiling-functions-consumer: <<: *snuba_defaults command: rust-consumer --storage functions_raw --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset + profiles: + - feature-complete snuba-spans-consumer: <<: *snuba_defaults command: rust-consumer --storage spans --consumer-group snuba-spans-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset + profiles: + - feature-complete symbolicator: <<: *restart_policy image: "$SYMBOLICATOR_IMAGE" @@ -348,6 +403,24 @@ services: - "-c" # Courtesy of https://unix.stackexchange.com/a/234089/108960 - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' + profiles: + - errors-only + web-feature-complete: + <<: *sentry_defaults-feature-complete + ulimits: + nofile: + soft: 4096 + hard: 4096 + healthcheck: + <<: *healthcheck_defaults + test: + - "CMD" + - "/bin/bash" + - "-c" + # Courtesy of https://unix.stackexchange.com/a/234089/108960 + - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' + profiles: + - feature-complete cron: <<: *sentry_defaults command: run cron @@ -360,57 +433,90 @@ services: attachments-consumer: <<: *sentry_defaults command: run consumer ingest-attachments --consumer-group ingest-consumer + post-process-forwarder-errors: + <<: *sentry_defaults + command: run consumer --no-strict-offset-reset post-process-forwarder-errors --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-commit-log --synchronize-commit-group=snuba-consumers + subscription-consumer-events: + <<: *sentry_defaults + command: run consumer events-subscription-results --consumer-group query-subscription-consumer + ############################################## + ## Feature Complete Sentry Ingest Consumers ## + ############################################## transactions-consumer: <<: *sentry_defaults command: run consumer ingest-transactions --consumer-group ingest-consumer + profiles: + - feature-complete metrics-consumer: <<: *sentry_defaults command: run consumer ingest-metrics --consumer-group metrics-consumer + profiles: + - feature-complete generic-metrics-consumer: <<: *sentry_defaults command: run consumer ingest-generic-metrics --consumer-group generic-metrics-consumer + profiles: + - feature-complete billing-metrics-consumer: <<: *sentry_defaults command: run consumer billing-metrics-consumer --consumer-group billing-metrics-consumer + profiles: + - feature-complete ingest-replay-recordings: <<: *sentry_defaults command: run consumer ingest-replay-recordings --consumer-group ingest-replay-recordings + profiles: + - feature-complete ingest-occurrences: <<: *sentry_defaults command: run consumer ingest-occurrences --consumer-group ingest-occurrences + profiles: + - feature-complete ingest-profiles: <<: *sentry_defaults command: run consumer ingest-profiles --consumer-group ingest-profiles + profiles: + - feature-complete ingest-monitors: <<: *sentry_defaults command: run consumer ingest-monitors --consumer-group ingest-monitors + profiles: + - feature-complete monitors-clock-tick: <<: *sentry_defaults command: run consumer monitors-clock-tick --consumer-group monitors-clock-tick + profiles: + - feature-complete monitors-clock-tasks: <<: *sentry_defaults command: run consumer monitors-clock-tasks --consumer-group monitors-clock-tasks - post-process-forwarder-errors: - <<: *sentry_defaults - command: run consumer --no-strict-offset-reset post-process-forwarder-errors --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-commit-log --synchronize-commit-group=snuba-consumers + profiles: + - feature-complete post-process-forwarder-transactions: <<: *sentry_defaults command: run consumer --no-strict-offset-reset post-process-forwarder-transactions --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group transactions_group + profiles: + - feature-complete post-process-forwarder-issue-platform: <<: *sentry_defaults command: run consumer --no-strict-offset-reset post-process-forwarder-issue-platform --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-generic-events-commit-log --synchronize-commit-group generic_events_group - subscription-consumer-events: - <<: *sentry_defaults - command: run consumer events-subscription-results --consumer-group query-subscription-consumer + profiles: + - feature-complete subscription-consumer-transactions: <<: *sentry_defaults command: run consumer transactions-subscription-results --consumer-group query-subscription-consumer + profiles: + - feature-complete subscription-consumer-metrics: <<: *sentry_defaults command: run consumer metrics-subscription-results --consumer-group query-subscription-consumer + profiles: + - feature-complete subscription-consumer-generic-metrics: <<: *sentry_defaults command: run consumer generic-metrics-subscription-results --consumer-group query-subscription-consumer + profiles: + - feature-complete sentry-cleanup: <<: *sentry_defaults image: sentry-cleanup-self-hosted-local @@ -434,6 +540,8 @@ services: depends_on: - web - relay + profiles: + - errors-only relay: <<: *restart_policy image: "$RELAY_IMAGE" @@ -453,6 +561,45 @@ services: <<: *depends_on-healthy web: <<: *depends_on-healthy + profiles: + - errors-only + nginx-feature-complete: + <<: *restart_policy + ports: + - "$SENTRY_BIND:80/tcp" + image: "nginx:1.25.4-alpine" + volumes: + - type: bind + read_only: true + source: ./nginx + target: /etc/nginx + - sentry-nginx-cache:/var/cache/nginx + depends_on: + - web-feature-complete + - relay-feature-complete + profiles: + - feature-complete + relay-feature-complete: + <<: *restart_policy + image: "$RELAY_IMAGE" + volumes: + - type: bind + read_only: true + source: ./relay + target: /work/.relay + - type: bind + read_only: true + source: ./geoip + target: /geoip + depends_on: + kafka: + <<: *depends_on-healthy + redis: + <<: *depends_on-healthy + web-feature-complete: + <<: *depends_on-healthy + profiles: + - feature-complete vroom: <<: *restart_policy image: "$VROOM_IMAGE" @@ -466,6 +613,8 @@ services: depends_on: kafka: <<: *depends_on-healthy + profiles: + - feature-complete vroom-cleanup: <<: *restart_policy image: vroom-cleanup-self-hosted-local @@ -481,6 +630,8 @@ services: command: '"0 0 * * * find /var/lib/sentry-profiles -type f -mtime +$SENTRY_EVENT_RETENTION_DAYS -delete"' volumes: - sentry-vroom:/var/lib/sentry-profiles + profiles: + - feature-complete volumes: # These store application data that should persist across restarts. From 36e286b005c7718fcfcd9d2f352a7020cbb8e016 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Tue, 9 Jul 2024 13:55:13 -0700 Subject: [PATCH 02/10] change to feature complete version of self-hosted --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index ccfc7594e6..430cdac1d5 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ COMPOSE_PROJECT_NAME=sentry-self-hosted -COMPOSE_PROFILES=errors-only +COMPOSE_PROFILES=feature-complete SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more From 92f11de067b3763b18847bf5f339b5185fe3d4f7 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Tue, 9 Jul 2024 13:58:53 -0700 Subject: [PATCH 03/10] add errors only option to config --- sentry/sentry.conf.example.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 5213d1e06a..51193bcaa7 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -362,3 +362,6 @@ def get_internal_network(): # https://docs.djangoproject.com/en/4.2/ref/settings/#std-setting-CSRF_TRUSTED_ORIGINS # CSRF_TRUSTED_ORIGINS = ["https://example.com", "http://127.0.0.1:9000"] + +# If you would like to use self-hosted Sentry with only errors enabled, please set this +SENTRY_SELF_HOSTED_ERRORS_ONLY = False From 005c4931a1c81713f94024e56339c9cc8c655b77 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Tue, 9 Jul 2024 15:25:30 -0700 Subject: [PATCH 04/10] check the output of docker compose up --- .github/workflows/test.yml | 1 + docker-compose.yml | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e565392c89..170c46f969 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -134,6 +134,7 @@ jobs: - name: Integration Test run: | + docker compose up -d if [ "${{ matrix.compose_version }}" = "v2.19.0" ]; then pytest --reruns 3 --cov --junitxml=junit.xml _integration-test/ --customizations=${{ matrix.customizations }} else diff --git a/docker-compose.yml b/docker-compose.yml index f4433bf900..f191b5a942 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -390,7 +390,7 @@ services: volumes: - "sentry-symbolicator:/data" web: - <<: *sentry_defaults + <<: *sentry_defaults-feature-complete ulimits: nofile: soft: 4096 @@ -404,9 +404,9 @@ services: # Courtesy of https://unix.stackexchange.com/a/234089/108960 - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' profiles: - - errors-only - web-feature-complete: - <<: *sentry_defaults-feature-complete + - feature-complete + web-errors-only: + <<: *sentry_defaults ulimits: nofile: soft: 4096 @@ -420,7 +420,7 @@ services: # Courtesy of https://unix.stackexchange.com/a/234089/108960 - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' profiles: - - feature-complete + - errors-only cron: <<: *sentry_defaults command: run cron @@ -541,7 +541,7 @@ services: - web - relay profiles: - - errors-only + - feature-complete relay: <<: *restart_policy image: "$RELAY_IMAGE" @@ -562,8 +562,8 @@ services: web: <<: *depends_on-healthy profiles: - - errors-only - nginx-feature-complete: + - feature-complete + nginx-errors-only: <<: *restart_policy ports: - "$SENTRY_BIND:80/tcp" @@ -575,11 +575,11 @@ services: target: /etc/nginx - sentry-nginx-cache:/var/cache/nginx depends_on: - - web-feature-complete - - relay-feature-complete + - web-errors-only + - relay-errors-only profiles: - - feature-complete - relay-feature-complete: + - errors-only + relay-errors-only: <<: *restart_policy image: "$RELAY_IMAGE" volumes: @@ -596,10 +596,10 @@ services: <<: *depends_on-healthy redis: <<: *depends_on-healthy - web-feature-complete: + web-errors-only: <<: *depends_on-healthy profiles: - - feature-complete + - errors-only vroom: <<: *restart_policy image: "$VROOM_IMAGE" From 6244c9b87450f2fff7d3880f8a07403a192f6846 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Tue, 9 Jul 2024 15:33:32 -0700 Subject: [PATCH 05/10] remove docker compose up in test yml --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 170c46f969..e565392c89 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -134,7 +134,6 @@ jobs: - name: Integration Test run: | - docker compose up -d if [ "${{ matrix.compose_version }}" = "v2.19.0" ]; then pytest --reruns 3 --cov --junitxml=junit.xml _integration-test/ --customizations=${{ matrix.customizations }} else From 414ba300e0f780ac88a163d9f6c00cf0be29a1b0 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Wed, 10 Jul 2024 15:28:50 -0700 Subject: [PATCH 06/10] much cleaner way of setting up docker compose file --- .env | 1 + docker-compose.yml | 65 +++-------------------------------- sentry/sentry.conf.example.py | 4 ++- 3 files changed, 9 insertions(+), 61 deletions(-) diff --git a/.env b/.env index 430cdac1d5..cde410dedf 100644 --- a/.env +++ b/.env @@ -1,4 +1,5 @@ COMPOSE_PROJECT_NAME=sentry-self-hosted +SENTRY_SELF_HOSTED_ERRORS_ONLY=false COMPOSE_PROFILES=feature-complete SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND diff --git a/docker-compose.yml b/docker-compose.yml index f191b5a942..7025b00d57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,11 +108,15 @@ x-sentry-defaults-feature-complete: &sentry_defaults-feature-complete <<: *depends_on-default vroom: <<: *depends_on-default + profiles: + - feature-complete x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: clickhouse: <<: *depends_on-healthy + zookeeper: + <<: *depends_on-default kafka: <<: *depends_on-healthy redis: @@ -390,7 +394,7 @@ services: volumes: - "sentry-symbolicator:/data" web: - <<: *sentry_defaults-feature-complete + <<: [*sentry_defaults, *sentry_defaults-feature-complete] ulimits: nofile: soft: 4096 @@ -403,24 +407,6 @@ services: - "-c" # Courtesy of https://unix.stackexchange.com/a/234089/108960 - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' - profiles: - - feature-complete - web-errors-only: - <<: *sentry_defaults - ulimits: - nofile: - soft: 4096 - hard: 4096 - healthcheck: - <<: *healthcheck_defaults - test: - - "CMD" - - "/bin/bash" - - "-c" - # Courtesy of https://unix.stackexchange.com/a/234089/108960 - - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' - profiles: - - errors-only cron: <<: *sentry_defaults command: run cron @@ -540,8 +526,6 @@ services: depends_on: - web - relay - profiles: - - feature-complete relay: <<: *restart_policy image: "$RELAY_IMAGE" @@ -561,45 +545,6 @@ services: <<: *depends_on-healthy web: <<: *depends_on-healthy - profiles: - - feature-complete - nginx-errors-only: - <<: *restart_policy - ports: - - "$SENTRY_BIND:80/tcp" - image: "nginx:1.25.4-alpine" - volumes: - - type: bind - read_only: true - source: ./nginx - target: /etc/nginx - - sentry-nginx-cache:/var/cache/nginx - depends_on: - - web-errors-only - - relay-errors-only - profiles: - - errors-only - relay-errors-only: - <<: *restart_policy - image: "$RELAY_IMAGE" - volumes: - - type: bind - read_only: true - source: ./relay - target: /work/.relay - - type: bind - read_only: true - source: ./geoip - target: /geoip - depends_on: - kafka: - <<: *depends_on-healthy - redis: - <<: *depends_on-healthy - web-errors-only: - <<: *depends_on-healthy - profiles: - - errors-only vroom: <<: *restart_policy image: "$VROOM_IMAGE" diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 51193bcaa7..08b52db21c 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -364,4 +364,6 @@ def get_internal_network(): # CSRF_TRUSTED_ORIGINS = ["https://example.com", "http://127.0.0.1:9000"] # If you would like to use self-hosted Sentry with only errors enabled, please set this -SENTRY_SELF_HOSTED_ERRORS_ONLY = False +SENTRY_SELF_HOSTED_ERRORS_ONLY = env( + "SENTRY_SELF_HOSTED_ERRORS_ONLY", "false" +).lower() in ["true", "1"] From 417886ffa5ebc8c9d463616a6add6dbed3857b97 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Mon, 15 Jul 2024 13:34:27 -0700 Subject: [PATCH 07/10] reduce depends on clauses --- docker-compose.yml | 50 +--------------------------------------------- 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7025b00d57..efecaadb0c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,8 +28,6 @@ x-sentry-defaults: &sentry_defaults <<: *depends_on-healthy kafka: <<: *depends_on-healthy - zookeeper: - <<: *depends_on-default postgres: <<: *depends_on-healthy memcached: @@ -40,18 +38,6 @@ x-sentry-defaults: &sentry_defaults <<: *depends_on-default symbolicator: <<: *depends_on-default - snuba-errors-consumer: - <<: *depends_on-default - snuba-outcomes-consumer: - <<: *depends_on-default - snuba-outcomes-billing-consumer: - <<: *depends_on-default - snuba-subscription-consumer-events: - <<: *depends_on-default - snuba-replacer: - <<: *depends_on-default - snuba-group-attributes-consumer: - <<: *depends_on-default entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] environment: @@ -78,45 +64,11 @@ x-sentry-defaults: &sentry_defaults - "./sentry:/etc/sentry" - "./geoip:/geoip:ro" - "./certificates:/usr/local/share/ca-certificates:ro" -x-sentry-defaults-feature-complete: &sentry_defaults-feature-complete - <<: *sentry_defaults - depends_on: - <<: *sentry_depends_on - snuba-transactions-consumer: - <<: *depends_on-default - snuba-replays-consumer: - <<: *depends_on-default - snuba-issue-occurrence-consumer: - <<: *depends_on-default - snuba-metrics-consumer: - <<: *depends_on-default - snuba-generic-metrics-distributions-consumer: - <<: *depends_on-default - snuba-generic-metrics-sets-consumer: - <<: *depends_on-default - snuba-generic-metrics-counters-consumer: - <<: *depends_on-default - snuba-subscription-consumer-transactions: - <<: *depends_on-default - snuba-subscription-consumer-metrics: - <<: *depends_on-default - snuba-profiling-profiles-consumer: - <<: *depends_on-default - snuba-profiling-functions-consumer: - <<: *depends_on-default - snuba-spans-consumer: - <<: *depends_on-default - vroom: - <<: *depends_on-default - profiles: - - feature-complete x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: clickhouse: <<: *depends_on-healthy - zookeeper: - <<: *depends_on-default kafka: <<: *depends_on-healthy redis: @@ -394,7 +346,7 @@ services: volumes: - "sentry-symbolicator:/data" web: - <<: [*sentry_defaults, *sentry_defaults-feature-complete] + <<: *sentry_defaults ulimits: nofile: soft: 4096 From a7a0908a2f58c7ef9aa2730badbbf9313127e4dd Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Mon, 15 Jul 2024 13:39:38 -0700 Subject: [PATCH 08/10] get rid of unused anchor --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index efecaadb0c..bbd007b515 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,7 @@ x-sentry-defaults: &sentry_defaults context: ./sentry args: - SENTRY_IMAGE - depends_on: &sentry_depends_on + depends_on: redis: <<: *depends_on-healthy kafka: From 251091ebcc5075871df7955e4eb06e4ee7956443 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Tue, 16 Jul 2024 11:19:11 -0700 Subject: [PATCH 09/10] fix --- docker-compose.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b5b57c0875..f21528c45c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -255,18 +255,6 @@ services: snuba-group-attributes-consumer: <<: *snuba_defaults command: rust-consumer --storage group_attributes --consumer-group snuba-group-attributes-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset - snuba-generic-metrics-distributions-consumer: - <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_distributions_raw --consumer-group snuba-gen-metrics-distributions-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset - snuba-generic-metrics-sets-consumer: - <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_sets_raw --consumer-group snuba-gen-metrics-sets-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset - snuba-generic-metrics-counters-consumer: - <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_counters_raw --consumer-group snuba-gen-metrics-counters-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset - snuba-generic-metrics-gauges-consumer: - <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_gauges_raw --consumer-group snuba-gen-metrics-gauges-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset snuba-replacer: <<: *snuba_defaults command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset @@ -322,6 +310,11 @@ services: command: rust-consumer --storage generic_metrics_counters_raw --consumer-group snuba-gen-metrics-counters-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset profiles: - feature-complete + snuba-generic-metrics-gauges-consumer: + <<: *snuba_defaults + command: rust-consumer --storage generic_metrics_gauges_raw --consumer-group snuba-gen-metrics-gauges-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + profiles: + - feature-complete snuba-profiling-profiles-consumer: <<: *snuba_defaults command: rust-consumer --storage profiles --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset From a0333f2a2057baedc4aa896d062977bc30ebf45a Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Thu, 18 Jul 2024 13:01:48 -0700 Subject: [PATCH 10/10] consolidate flags --- .env | 1 - docker-compose.yml | 1 + sentry/sentry.conf.example.py | 4 +--- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.env b/.env index cde410dedf..430cdac1d5 100644 --- a/.env +++ b/.env @@ -1,5 +1,4 @@ COMPOSE_PROJECT_NAME=sentry-self-hosted -SENTRY_SELF_HOSTED_ERRORS_ONLY=false COMPOSE_PROFILES=feature-complete SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND diff --git a/docker-compose.yml b/docker-compose.yml index f21528c45c..af7aab1d2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,6 +55,7 @@ x-sentry-defaults: &sentry_defaults GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR: *ca_bundle # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) + COMPOSE_PROFILES: SENTRY_EVENT_RETENTION_DAYS: SENTRY_MAIL_HOST: SENTRY_MAX_EXTERNAL_SOURCEMAP_SIZE: diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 3cb945a521..0f8d611700 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -365,6 +365,4 @@ def get_internal_network(): # CSRF_TRUSTED_ORIGINS = ["https://example.com", "http://127.0.0.1:9000"] # If you would like to use self-hosted Sentry with only errors enabled, please set this -SENTRY_SELF_HOSTED_ERRORS_ONLY = env( - "SENTRY_SELF_HOSTED_ERRORS_ONLY", "false" -).lower() in ["true", "1"] +SENTRY_SELF_HOSTED_ERRORS_ONLY = env("COMPOSE_PROFILES") == "feature-complete"