From acaafab8cafbed76b58349e38d2184f759b1e013 Mon Sep 17 00:00:00 2001 From: Marc Sanmiquel Date: Mon, 16 Dec 2024 08:51:53 +0100 Subject: [PATCH] examples: add ruby span profiling --- .../python-span-profiles.md | 2 +- .../trace-span-profiles/ruby-span-profiles.md | 6 + examples/tracing/ruby/.ruby-version | 1 + examples/tracing/ruby/Dockerfile | 12 ++ examples/tracing/ruby/Gemfile | 12 ++ examples/tracing/ruby/Gemfile.lock | 113 ++++++++++++++++++ examples/tracing/ruby/README.md | 56 +++++++++ examples/tracing/ruby/docker-compose.yml | 87 ++++++++++++++ .../datasources/pyroscope.yml | 31 +++++ .../plugins/explore-profiles.yml | 7 ++ examples/tracing/ruby/lib/bike/bike.rb | 5 + examples/tracing/ruby/lib/car/car.rb | 5 + examples/tracing/ruby/lib/scooter/scooter.rb | 5 + examples/tracing/ruby/lib/server.rb | 83 +++++++++++++ examples/tracing/ruby/lib/utility/utility.rb | 38 ++++++ examples/tracing/ruby/tempo/tempo.yml | 39 ++++++ 16 files changed, 501 insertions(+), 1 deletion(-) create mode 100644 examples/tracing/ruby/.ruby-version create mode 100644 examples/tracing/ruby/Dockerfile create mode 100644 examples/tracing/ruby/Gemfile create mode 100644 examples/tracing/ruby/Gemfile.lock create mode 100644 examples/tracing/ruby/README.md create mode 100644 examples/tracing/ruby/docker-compose.yml create mode 100644 examples/tracing/ruby/grafana-provisioning/datasources/pyroscope.yml create mode 100644 examples/tracing/ruby/grafana-provisioning/plugins/explore-profiles.yml create mode 100644 examples/tracing/ruby/lib/bike/bike.rb create mode 100644 examples/tracing/ruby/lib/car/car.rb create mode 100644 examples/tracing/ruby/lib/scooter/scooter.rb create mode 100644 examples/tracing/ruby/lib/server.rb create mode 100644 examples/tracing/ruby/lib/utility/utility.rb create mode 100644 examples/tracing/ruby/tempo/tempo.yml diff --git a/docs/sources/configure-client/trace-span-profiles/python-span-profiles.md b/docs/sources/configure-client/trace-span-profiles/python-span-profiles.md index adc24fbf7b..efe470a176 100644 --- a/docs/sources/configure-client/trace-span-profiles/python-span-profiles.md +++ b/docs/sources/configure-client/trace-span-profiles/python-span-profiles.md @@ -76,5 +76,5 @@ Refer to the [data source configuration documentation](https://grafana.com/docs/ ## Examples Check out these demo applications for span profiles: -- [Python example](https://github.com/grafana/pyroscope/tree/main/examples/tracing/tempo/python) +- [Python example](https://github.com/grafana/pyroscope/tree/main/examples/tracing/python) - [Other examples](https://github.com/grafana/pyroscope/tree/main/examples/tracing/tempo) in multiple languages diff --git a/docs/sources/configure-client/trace-span-profiles/ruby-span-profiles.md b/docs/sources/configure-client/trace-span-profiles/ruby-span-profiles.md index 4086e134de..402481ae8f 100644 --- a/docs/sources/configure-client/trace-span-profiles/ruby-span-profiles.md +++ b/docs/sources/configure-client/trace-span-profiles/ruby-span-profiles.md @@ -87,3 +87,9 @@ The profile type or app must be selected for the query to be valid. Grafana does ## Examples Check out the [examples](https://github.com/grafana/pyroscope/tree/main/examples/tracing/tempo) directory for a complete demo application that shows tracing integration features. + +## Examples + +Check out these demo applications for span profiles: +- [Ruby example](https://github.com/grafana/pyroscope/tree/main/examples/tracing/ruby) +- [Other examples](https://github.com/grafana/pyroscope/tree/main/examples/tracing/tempo) in multiple languages diff --git a/examples/tracing/ruby/.ruby-version b/examples/tracing/ruby/.ruby-version new file mode 100644 index 0000000000..be94e6f53d --- /dev/null +++ b/examples/tracing/ruby/.ruby-version @@ -0,0 +1 @@ +3.2.2 diff --git a/examples/tracing/ruby/Dockerfile b/examples/tracing/ruby/Dockerfile new file mode 100644 index 0000000000..e0b302d8f4 --- /dev/null +++ b/examples/tracing/ruby/Dockerfile @@ -0,0 +1,12 @@ +FROM ruby:3.2.2 + +WORKDIR /opt/app + +COPY Gemfile ./Gemfile +COPY Gemfile.lock ./Gemfile.lock +# RUN bundle config set --local deployment true +RUN bundle install + +COPY lib ./lib + +CMD [ "ruby", "lib/server.rb" ] diff --git a/examples/tracing/ruby/Gemfile b/examples/tracing/ruby/Gemfile new file mode 100644 index 0000000000..4710a022b9 --- /dev/null +++ b/examples/tracing/ruby/Gemfile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } + +gem 'pyroscope', '= 0.5.12' +gem "sinatra", "~> 2.1" +gem "thin", "~> 1.8" +gem 'pyroscope-otel' +gem 'opentelemetry-sdk' +gem 'opentelemetry-exporter-otlp' diff --git a/examples/tracing/ruby/Gemfile.lock b/examples/tracing/ruby/Gemfile.lock new file mode 100644 index 0000000000..6282c7dc6a --- /dev/null +++ b/examples/tracing/ruby/Gemfile.lock @@ -0,0 +1,113 @@ +GEM + remote: https://rubygems.org/ + specs: + bigdecimal (3.1.8) + daemons (1.4.1) + eventmachine (1.2.7) + ffi (1.17.0) + ffi (1.17.0-aarch64-linux-gnu) + ffi (1.17.0-aarch64-linux-musl) + ffi (1.17.0-arm-linux-gnu) + ffi (1.17.0-arm-linux-musl) + ffi (1.17.0-arm64-darwin) + ffi (1.17.0-x86-linux-gnu) + ffi (1.17.0-x86-linux-musl) + ffi (1.17.0-x86_64-darwin) + ffi (1.17.0-x86_64-linux-gnu) + ffi (1.17.0-x86_64-linux-musl) + google-protobuf (4.29.1) + bigdecimal + rake (>= 13) + google-protobuf (4.29.1-aarch64-linux) + bigdecimal + rake (>= 13) + google-protobuf (4.29.1-arm64-darwin) + bigdecimal + rake (>= 13) + google-protobuf (4.29.1-x86-linux) + bigdecimal + rake (>= 13) + google-protobuf (4.29.1-x86_64-darwin) + bigdecimal + rake (>= 13) + google-protobuf (4.29.1-x86_64-linux) + bigdecimal + rake (>= 13) + googleapis-common-protos-types (1.16.0) + google-protobuf (>= 3.18, < 5.a) + mustermann (2.0.2) + ruby2_keywords (~> 0.0.1) + opentelemetry-api (1.1.0) + opentelemetry-common (0.21.0) + opentelemetry-api (~> 1.0) + opentelemetry-exporter-otlp (0.29.1) + google-protobuf (>= 3.18) + googleapis-common-protos-types (~> 1.3) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-sdk (~> 1.2) + opentelemetry-semantic_conventions + opentelemetry-registry (0.3.1) + opentelemetry-api (~> 1.1) + opentelemetry-sdk (1.6.0) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-registry (~> 0.2) + opentelemetry-semantic_conventions + opentelemetry-semantic_conventions (1.10.1) + opentelemetry-api (~> 1.0) + pyroscope (0.5.12) + ffi + pyroscope (0.5.12-aarch64-linux) + ffi + pyroscope (0.5.12-arm64-darwin) + ffi + pyroscope (0.5.12-x86_64-darwin) + ffi + pyroscope (0.5.12-x86_64-linux) + ffi + pyroscope-otel (0.1.1) + opentelemetry-api (~> 1.1.0) + pyroscope (~> 0.5.1) + rack (2.2.10) + rack-protection (2.2.4) + rack + rake (13.2.1) + ruby2_keywords (0.0.5) + sinatra (2.2.4) + mustermann (~> 2.0) + rack (~> 2.2) + rack-protection (= 2.2.4) + tilt (~> 2.0) + thin (1.8.2) + daemons (~> 1.0, >= 1.0.9) + eventmachine (~> 1.0, >= 1.0.4) + rack (>= 1, < 3) + tilt (2.4.0) + +PLATFORMS + aarch64-linux + aarch64-linux-gnu + aarch64-linux-musl + arm-linux-gnu + arm-linux-musl + arm64-darwin + ruby + x86-linux + x86-linux-gnu + x86-linux-musl + x86_64-darwin + x86_64-linux + x86_64-linux-gnu + x86_64-linux-musl + +DEPENDENCIES + opentelemetry-exporter-otlp + opentelemetry-sdk + pyroscope (= 0.5.12) + pyroscope-otel + sinatra (~> 2.1) + thin (~> 1.8) + +BUNDLED WITH + 2.5.23 diff --git a/examples/tracing/ruby/README.md b/examples/tracing/ruby/README.md new file mode 100644 index 0000000000..3fd8ca4139 --- /dev/null +++ b/examples/tracing/ruby/README.md @@ -0,0 +1,56 @@ +# Span Profiles with Grafana Tempo and Pyroscope + +The docker compose consists of: +- Three Ruby Rideshare App instances (us-east, eu-north, ap-south regions) +- Tempo for trace collection +- Pyroscope for continuous profiling +- Grafana for visualization +- Load Generator for simulating traffic + +For a detailed guide about Ruby span profiles configuration, refer to the docs [Pyroscope documentation](https://grafana.com/docs/pyroscope/latest/configure-client/trace-span-profiles/ruby-span-profiles/). + +The `rideshare` app generates traces and profiling data that should be available in Grafana. +Pyroscope and Tempo datasources are provisioned automatically. + +### Build and run + +The project can be run locally with the following commands: + +```shell +# Pull latest pyroscope and grafana images: +docker pull grafana/pyroscope:latest +docker pull grafana/grafana:latest + +bundle install + +docker compose up +``` +The load generator will automatically start sending requests to all regional instances. + +### Viewing Traces and Profiles + +Navigate to the [Explore page](http://localhost:3000/explore?schemaVersion=1&panes=%7B%22yM9%22:%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22tempo%22,%22uid%22:%22tempo%22%7D,%22queryType%22:%22traceqlSearch%22,%22limit%22:20,%22tableType%22:%22traces%22,%22filters%22:%5B%7B%22id%22:%22e73a615e%22,%22operator%22:%22%3D%22,%22scope%22:%22span%22%7D,%7B%22id%22:%22service-name%22,%22tag%22:%22service.name%22,%22operator%22:%22%3D%22,%22scope%22:%22resource%22,%22value%22:%5B%22rideshare.ruby.push.app%22%5D,%22valueType%22:%22string%22%7D%5D%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1), select a trace and click on a span that has a linked profile: + +![image](https://github.com/grafana/otel-profiling-go/assets/12090599/31e33cd1-818b-4116-b952-c9ec7b1fb593) + +By default, only the root span gets labeled (the first span created locally): such spans are marked with the link icon +and have `pyroscope.profile.id` attribute set to the corresponding span ID. +Please note that presence of the attribute does not necessarily +indicate that the span has a profile: stack trace samples might not be collected, if the utilized CPU time is +less than the sample interval (10ms). + +### Grafana Tempo configuration + +In order to correlate trace spans with profiling data, the Tempo datasource should have the following configured: +- The profiling data source +- Tags to use when making profiling queries + +![image](https://github.com/grafana/pyroscope/assets/12090599/380ac574-a298-440d-acfb-7bc0935a3a7c) + +While tags are optional, configuring them is highly recommended for optimizing query performance. +In our example, we configured the `service.name` tag for use in Pyroscope queries as the `service_name` label. +This configuration restricts the data set for lookup, ensuring that queries remain +consistently fast. Note that the tags you configure must be present in the span attributes or resources +for a trace to profiles span link to appear. + +Please refer to our [documentation](https://grafana.com/docs/grafana/next/datasources/tempo/configure-tempo-data-source/#trace-to-profiles) for more details. diff --git a/examples/tracing/ruby/docker-compose.yml b/examples/tracing/ruby/docker-compose.yml new file mode 100644 index 0000000000..0a1b872598 --- /dev/null +++ b/examples/tracing/ruby/docker-compose.yml @@ -0,0 +1,87 @@ +services: + pyroscope: + image: grafana/pyroscope + ports: + - "4040:4040" + + us-east: + ports: + - "5000" + hostname: us-east + environment: &env + OTLP_URL: tempo:4318 + OTLP_INSECURE: 1 + DEBUG_LOGGER: 1 + OTEL_LOG_LEVEL: debug + OTEL_TRACES_EXPORTER: otlp + OTEL_EXPORTER_OTLP_ENDPOINT: "http://tempo:4318" + OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf" + OTEL_SDK_DEBUG: "true" + OTEL_SERVICE_NAME: rideshare.ruby.push.app + OTEL_METRICS_EXPORTER: none + OTEL_TRACES_SAMPLER: always_on + OTEL_PROPAGATORS: tracecontext + REGION: us-east + PYROSCOPE_LABELS: hostname=us-east + PYROSCOPE_SERVER_ADDRESS: http://pyroscope:4040 + build: + context: . + + eu-north: + ports: + - "5000" + hostname: eu-north + environment: + <<: *env + REGION: eu-north + PYROSCOPE_LABELS: hostname=eu-north + build: + context: . + + ap-south: + ports: + - "5000" + hostname: ap-south + environment: + <<: *env + REGION: ap-south + PYROSCOPE_LABELS: hostname=ap-south + build: + context: . + + load-generator: + environment: *env + build: + context: ../../language-sdk-instrumentation/golang-push/rideshare + dockerfile: Dockerfile.load-generator + command: + - ./loadgen + - http://us-east:5000 + - http://eu-north:5000 + - http://ap-south:5000 + + grafana: + image: grafana/grafana:latest + environment: + - GF_INSTALL_PLUGINS=grafana-pyroscope-app + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + - GF_FEATURE_TOGGLES_ENABLE=traceToProfiles tracesEmbeddedFlameGraph + volumes: + - ./grafana-provisioning:/etc/grafana/provisioning + ports: + - "3000:3000" + + tempo: + image: grafana/tempo:latest + command: [ "-config.file=/etc/tempo.yml" ] + volumes: + - ./tempo/tempo.yml:/etc/tempo.yml + ports: + - "14268:14268" # jaeger ingest + - "3200:3200" # tempo + - "9095:9095" # tempo grpc + - "4317:4317" # otlp grpc + - "4318:4318" # otlp http + - "9411:9411" # zipkin diff --git a/examples/tracing/ruby/grafana-provisioning/datasources/pyroscope.yml b/examples/tracing/ruby/grafana-provisioning/datasources/pyroscope.yml new file mode 100644 index 0000000000..25c0e11477 --- /dev/null +++ b/examples/tracing/ruby/grafana-provisioning/datasources/pyroscope.yml @@ -0,0 +1,31 @@ +--- +apiVersion: 1 +datasources: + - name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://tempo:3200 + basicAuth: false + isDefault: true + version: 1 + editable: false + apiVersion: 1 + uid: tempo + jsonData: + httpMethod: GET + serviceMap: + datasourceUid: prometheus + tracesToProfiles: + customQuery: false + datasourceUid: "pyroscope" + profileTypeId: "process_cpu:cpu:nanoseconds:cpu:nanoseconds" + tags: + - key: "service.name" + value: "service_name" + - uid: pyroscope + type: grafana-pyroscope-datasource + name: Pyroscope + url: http://pyroscope:4040 + jsonData: + keepCookies: [pyroscope_git_session] diff --git a/examples/tracing/ruby/grafana-provisioning/plugins/explore-profiles.yml b/examples/tracing/ruby/grafana-provisioning/plugins/explore-profiles.yml new file mode 100644 index 0000000000..0b979b4c1c --- /dev/null +++ b/examples/tracing/ruby/grafana-provisioning/plugins/explore-profiles.yml @@ -0,0 +1,7 @@ +--- +apiVersion: 1 +apps: + - type: grafana-pyroscope-app + jsonData: + backendUrl: http://pyroscope:4040 + secureJsonData: diff --git a/examples/tracing/ruby/lib/bike/bike.rb b/examples/tracing/ruby/lib/bike/bike.rb new file mode 100644 index 0000000000..492cfae42b --- /dev/null +++ b/examples/tracing/ruby/lib/bike/bike.rb @@ -0,0 +1,5 @@ +require_relative '../utility/utility' + +def order_bike(search_radius) + find_nearest_vehicle(search_radius, "bike") +end diff --git a/examples/tracing/ruby/lib/car/car.rb b/examples/tracing/ruby/lib/car/car.rb new file mode 100644 index 0000000000..452d78d2df --- /dev/null +++ b/examples/tracing/ruby/lib/car/car.rb @@ -0,0 +1,5 @@ +require_relative '../utility/utility' + +def order_car(search_radius) + find_nearest_vehicle(search_radius, "car") +end diff --git a/examples/tracing/ruby/lib/scooter/scooter.rb b/examples/tracing/ruby/lib/scooter/scooter.rb new file mode 100644 index 0000000000..d9c13ac9da --- /dev/null +++ b/examples/tracing/ruby/lib/scooter/scooter.rb @@ -0,0 +1,5 @@ +require_relative '../utility/utility' + +def order_scooter(search_radius) + find_nearest_vehicle(search_radius, "scooter") +end diff --git a/examples/tracing/ruby/lib/server.rb b/examples/tracing/ruby/lib/server.rb new file mode 100644 index 0000000000..5e92ec332e --- /dev/null +++ b/examples/tracing/ruby/lib/server.rb @@ -0,0 +1,83 @@ +require "sinatra" +require "thin" +require "pyroscope" +require "pyroscope/otel" +require "opentelemetry-sdk" +require 'opentelemetry-exporter-otlp' +require 'opentelemetry/trace/propagation/trace_context' +require_relative 'scooter/scooter' +require_relative 'bike/bike' +require_relative 'car/car' + +app_name = ENV.fetch("PYROSCOPE_APPLICATION_NAME", "rideshare.ruby.push.app") +pyroscope_server_address = ENV.fetch("PYROSCOPE_SERVER_ADDRESS", "http://pyroscope:4040") + +Pyroscope.configure do |config| + config.app_name = app_name + config.server_address = pyroscope_server_address + config.tags = { + "region": ENV["REGION"], + } +end + +OpenTelemetry::SDK.configure do |c| + c.add_span_processor Pyroscope::Otel::SpanProcessor.new("#{app_name}.cpu", pyroscope_server_address) + + c.add_span_processor( + OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + OpenTelemetry::Exporter::OTLP::Exporter.new( + endpoint: 'http://tempo:4318/v1/traces' + ) + ) + ) + +end + +# Extract trace context from load generator requests to link our handler spans with the parent +# load generator trace, creating a complete distributed trace across both services. +before do + if (traceparent = request.env['HTTP_TRACEPARENT']) + # Parse traceparent: version-traceid-spanid-flags + _version, trace_id_hex, parent_span_id_hex, _flags = traceparent.split('-') + + # Get the propagator and carrier + carrier = { 'traceparent' => traceparent } + + # Extract context using the propagator + @extracted_context = OpenTelemetry.propagation.extract(carrier) + end +end + +tracer = OpenTelemetry.tracer_provider.tracer('my-tracer') + +get "/bike" do + OpenTelemetry::Context.with_current(@extracted_context) do + tracer.in_span("BikeHandler") do |span| + order_bike(0.4) + "

Bike ordered

" + end + end +end + +get "/scooter" do + OpenTelemetry::Context.with_current(@extracted_context) do + tracer.in_span("ScooterHandler") do |span| + order_scooter(0.6) + "

scooter ordered

" + end + end +end + +get "/car" do + OpenTelemetry::Context.with_current(@extracted_context) do + tracer.in_span("CarHandler") do |span| + order_car(0.8) + "

car ordered

" + end + end +end + +set :bind, '0.0.0.0' +set :port, ENV["RIDESHARE_LISTEN_PORT"] || 5000 + +run Sinatra::Application.run! diff --git a/examples/tracing/ruby/lib/utility/utility.rb b/examples/tracing/ruby/lib/utility/utility.rb new file mode 100644 index 0000000000..8f1a5edaea --- /dev/null +++ b/examples/tracing/ruby/lib/utility/utility.rb @@ -0,0 +1,38 @@ +require "pyroscope" + +def mutex_lock(n) + i = 0 + start_time = Time.new + while Time.new - start_time < n * 10 do + i += 1 + end +end + +def check_driver_availability(n) + i = 0 + start_time = Time.new + while Time.new - start_time < n / 2 do + i += 1 + end + + # Every 4 minutes this will artificially create make requests in eu-north region slow + # this is just for demonstration purposes to show how performance impacts show up in the + # flamegraph + current_time = Time.now + current_minute = current_time.strftime('%M').to_i + force_mutex_lock = (current_minute * 4 % 8) == 0 + + mutex_lock(n) if ENV["REGION"] == "eu-north" and force_mutex_lock +end + +def find_nearest_vehicle(n, vehicle) + Pyroscope.tag_wrapper({ "vehicle" => vehicle }) do + i = 0 + start_time = Time.new + while Time.new - start_time < n do + i += 1 + end + + check_driver_availability(n) if vehicle == "car" + end +end diff --git a/examples/tracing/ruby/tempo/tempo.yml b/examples/tracing/ruby/tempo/tempo.yml new file mode 100644 index 0000000000..7a558043a3 --- /dev/null +++ b/examples/tracing/ruby/tempo/tempo.yml @@ -0,0 +1,39 @@ +server: + http_listen_port: 3200 + +query_frontend: + search: + duration_slo: 5s + throughput_bytes_slo: 1.073741824e+09 + trace_by_id: + duration_slo: 5s + +distributor: + receivers: # this configuration will listen on all ports and protocols that tempo is capable of. + jaeger: # the receives all come from the OpenTelemetry collector. more configuration information can + protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver + thrift_http: # + grpc: # for a production deployment you should only enable the receivers you need! + thrift_binary: + thrift_compact: + zipkin: + otlp: + protocols: + http: + grpc: + opencensus: + +ingester: + max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally + +compactor: + compaction: + block_retention: 1h # overall Tempo trace retention. set for demo purposes + +storage: + trace: + backend: local # backend configuration to use + wal: + path: /tmp/tempo/wal # where to store the wal locally + local: + path: /tmp/tempo/blocks