feat(spans): Ingest spans (#2861)

getsentry · Mar 11, 2024 · f64b0fb · f64b0fb
1 parent 746031d
commit f64b0fb
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 8 deletions.
diff --git a/_integration-test/fixtures/envelope-with-transaction b/_integration-test/fixtures/envelope-with-transaction
@@ -0,0 +1,3 @@
+{"event_id":"66578634d48d433db0ad52882d1efe5b","sent_at":"2023-05-17T14:54:31.057Z","sdk":{"name":"sentry.javascript.node","version":"7.46.0"},"trace":{"environment":"production","transaction":"fib: sourcemaps here","public_key":"05ab86aebbe14a24bcab62caa839cf27","trace_id":"33321bfbd5304bcc9663d1b53b08f84e","sample_rate":"1"}}
+{"type":"transaction"}
+{"contexts":{"trace":{"op":"test","span_id":"b38f2b24537c3858","trace_id":"33321bfbd5304bcc9663d1b53b08f84e"},"runtime":{"name":"node","version":"v16.16.0"},"app":{"app_start_time":"2023-05-17T14:54:27.678Z","app_memory":57966592},"os":{"kernel_version":"22.3.0","name":"macOS","version":"13.2","build":"22D49"},"device":{"boot_time":"2023-05-12T15:08:41.047Z","arch":"arm64","memory_size":34359738368,"free_memory":6861651968,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-US","timezone":"America/New_York"}},"spans":[],"start_timestamp":1684335267.744,"tags":{},"timestamp":1684335271.033,"transaction":"fib: sourcemaps here","type":"transaction","transaction_info":{"source":"custom"},"platform":"node","server_name":"TK6G745PW1.local","event_id":"66578634d48d433db0ad52882d1efe5b","environment":"production","sdk":{"integrations":["InboundFilters","FunctionToString","Console","Http","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","Modules","RequestData","LinkedErrors","ProfilingIntegration"],"name":"sentry.javascript.node","version":"7.46.0","packages":[{"name":"npm:@sentry/node","version":"7.46.0"}]},"debug_meta":{"images":[]},"modules":{}}
diff --git a/_integration-test/run.sh b/_integration-test/run.sh
@@ -113,12 +113,26 @@ echo "${_endgroup}"
 echo "${_group}Test that profiling work ..."
 echo "Sending a test profile..."
 PROFILE_FIXTURE_PATH="$(git rev-parse --show-toplevel)/_integration-test/fixtures/envelope-with-profile"
-curl -sf --data-binary @$PROFILE_FIXTURE_PATH -H 'Content-Type: application/x-sentry-envelope' -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" "$SENTRY_TEST_HOST/api/$PROJECT_ID/envelope/" -o /dev/null
+curl -sf --data-binary @$PROFILE_FIXTURE_PATH -H 'Content-Type: application/x-sentry-envelope' -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" "$SENTRY_TEST_HOST/api/$PROJECT_ID/envelope/"
 
 printf "Getting the test profile back"
 PROFILE_ID="$(jq -r -n --slurpfile profile $PROFILE_FIXTURE_PATH '$profile[4].event_id')"
-PROFILE_PATH="api/0/projects/sentry/sentry/profiling/raw_profiles/$PROFILE_ID/"
-timeout 60 bash -c 'until $(sentry_api_request "$PROFILE_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done'
+PROFILE_PATH="projects/sentry/sentry/profiling/raw_profiles/$PROFILE_ID/"
+timeout 60 bash -c 'until sentry_api_request "$PROFILE_PATH" -X GET -o /dev/null; do printf '.'; sleep 0.5; done'
+echo " got it!"
+echo "${_endgroup}"
+
+echo "${_group}Test we can extract spans from an event..."
+echo "Sending a test span..."
+SPAN_FIXTURE_PATH="$(git rev-parse --show-toplevel)/_integration-test/fixtures/envelope-with-transaction"
+curl -sf --data-binary @$PROFILE_FIXTURE_PATH -H 'Content-Type: application/x-sentry-envelope' -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" "$SENTRY_TEST_HOST/api/$PROJECT_ID/envelope/"
+
+printf "Getting a span back"
+TRACE_ID="$(jq -r -n --slurpfile span $SPAN_FIXTURE_PATH '$span[2].contexts.trace.trace_id')"
+SPAN_PATH="organizations/sentry/events/"
+SPAN_QUERY_PARAMS="-G --data-urlencode dataset=spansIndexed --data-urlencode field=id --data-urlencode project=1 --data-urlencode query=trace:$TRACE_ID --data-urlencode statsPeriod=1h"
+sleep 10
+sentry_api_request $SPAN_PATH -X GET $SPAN_QUERY_PARAMS | jq .data[] -e
 echo " got it!"
 echo "${_endgroup}"
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -303,6 +303,9 @@ services:
   snuba-profiling-functions-consumer:
     <<: *snuba_defaults
     command: rust-consumer --storage functions_raw --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --no-skip-write
+  snuba-spans-consumer:
+    <<: *snuba_defaults
+    command: rust-consumer --storage spans --consumer-group snuba-spans-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --no-skip-write
   symbolicator:
     <<: *restart_policy
     image: "$SYMBOLICATOR_IMAGE"

diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py
@@ -5,10 +5,12 @@
 
 BYTE_MULTIPLIER = 1024
 UNITS = ("K", "M", "G")
+
+
 def unit_text_to_bytes(text):
     unit = text[-1].upper()
     power = UNITS.index(unit) + 1
-    return float(text[:-1])*(BYTE_MULTIPLIER**power)
+    return float(text[:-1]) * (BYTE_MULTIPLIER**power)
 
 
 # Generously adapted from pynetlinux: https://github.com/rlisagor/pynetlinux/blob/e3f16978855c6649685f0c43d4c3fcf768427ae5/pynetlinux/ifconfig.py#L197-L223
@@ -114,7 +116,7 @@ def get_internal_network():
         "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache",
         "LOCATION": ["memcached:11211"],
         "TIMEOUT": 3600,
-        "OPTIONS": {"ignore_exc": True}
+        "OPTIONS": {"ignore_exc": True},
     }
 }
 
@@ -191,7 +193,9 @@ def get_internal_network():
 ###################
 
 SENTRY_RELEASE_HEALTH = "sentry.release_health.metrics.MetricsReleaseHealthBackend"
-SENTRY_RELEASE_MONITOR = "sentry.release_health.release_monitor.metrics.MetricReleaseMonitorBackend"
+SENTRY_RELEASE_MONITOR = (
+    "sentry.release_health.release_monitor.metrics.MetricReleaseMonitorBackend"
+)
 
 ##############
 # Web Server #
@@ -248,7 +252,7 @@ def get_internal_network():
 # Mail #
 ########
 
-SENTRY_OPTIONS["mail.list-namespace"] = env('SENTRY_MAIL_HOST', 'localhost')
+SENTRY_OPTIONS["mail.list-namespace"] = env("SENTRY_MAIL_HOST", "localhost")
 SENTRY_OPTIONS["mail.from"] = f"sentry@{SENTRY_OPTIONS['mail.list-namespace']}"
 
 ############
@@ -289,14 +293,33 @@ def get_internal_network():
             "projects:rate-limits",
             "projects:servicehooks",
         )
+        + (
+            "projects:span-metrics-extraction",
+            "organizations:starfish-browser-resource-module-image-view",
+            "organizations:starfish-browser-resource-module-ui",
+            "organizations:starfish-browser-webvitals",
+            "organizations:starfish-browser-webvitals-pageoverview-v2",
+            "organizations:starfish-browser-webvitals-use-backend-scores",
+            "organizations:performance-calculate-score-relay",
+            "organizations:starfish-browser-webvitals-replace-fid-with-inp",
+            "organizations:deprecate-fid-from-performance-score",
+            "organizations:performance-database-view",
+            "organizations:starfish-browser-webvitals",
+            "organizations:performance-screens-view",
+            "organizations:starfish-browser-resource-module-ui",
+            "organizations:starfish-browser-webvitals-pageoverview-v2",
+            "organizations:starfish-browser-webvitals-use-backend-scores",
+            "organizations:mobile-ttid-ttfd-contribution",
+            "organizations:starfish-mobile-appstart",
+        )  # starfish related flags
     }
 )
 
 #######################
 # MaxMind Integration #
 #######################
 
-GEOIP_PATH_MMDB = '/geoip/GeoLite2-City.mmdb'
+GEOIP_PATH_MMDB = "/geoip/GeoLite2-City.mmdb"
 
 #########################
 # Bitbucket Integration #