From 9fb7acec6bfb926a73221f48e5070290617fe66a Mon Sep 17 00:00:00 2001 From: David Elner Date: Thu, 4 Jan 2018 18:24:06 -0500 Subject: [PATCH 01/72] Added: GraphQL integration configuration. --- Appraisals | 1 + Rakefile | 19 +++++---- gemfiles/contrib.gemfile | 1 + lib/ddtrace/contrib/graphql/patcher.rb | 54 ++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 lib/ddtrace/contrib/graphql/patcher.rb diff --git a/Appraisals b/Appraisals index 4a6885ec2a9..98c01b0d8c0 100644 --- a/Appraisals +++ b/Appraisals @@ -115,6 +115,7 @@ if RUBY_VERSION >= '2.2.2' && RUBY_PLATFORM != 'java' appraise 'contrib' do gem 'elasticsearch-transport' gem 'mongo', '< 2.5' + gem 'graphql' gem 'grape' gem 'rack' gem 'rack-test' diff --git a/Rakefile b/Rakefile index 0cbe74f5a84..ad6ff84893e 100644 --- a/Rakefile +++ b/Rakefile @@ -108,18 +108,20 @@ namespace :test do end [ + :aws, :elasticsearch, - :http, - :redis, - :sinatra, - :sidekiq, - :rack, :faraday, :grape, - :aws, - :sucker_punch, + :graphql, + :http, :mongodb, - :resque + :resque, + :rack, + :redis, + :resque, + :sidekiq, + :sinatra, + :sucker_punch ].each do |contrib| Rake::TestTask.new(contrib) do |t| t.libs << %w[test lib] @@ -211,6 +213,7 @@ task :ci do sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:sinatra' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:rack' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:grape' + sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:graphql' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:faraday' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:aws' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:mongodb' diff --git a/gemfiles/contrib.gemfile b/gemfiles/contrib.gemfile index 72b1c3addae..d61a27cee90 100644 --- a/gemfiles/contrib.gemfile +++ b/gemfiles/contrib.gemfile @@ -5,6 +5,7 @@ source "https://rubygems.org" gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master" gem "elasticsearch-transport" gem "mongo", "< 2.5" +gem "graphql" gem "grape" gem "rack" gem "rack-test" diff --git a/lib/ddtrace/contrib/graphql/patcher.rb b/lib/ddtrace/contrib/graphql/patcher.rb new file mode 100644 index 00000000000..2b0f0adb922 --- /dev/null +++ b/lib/ddtrace/contrib/graphql/patcher.rb @@ -0,0 +1,54 @@ +require 'ddtrace/ext/app_types' +require 'ddtrace/ext/http' + +module Datadog + module Contrib + module GraphQL + # Provides instrumentation for `graphql` through the GraphQL tracing framework + module Patcher + include Base + register_as :graphql + option :tracer, default: Datadog.tracer + option :service_name, default: 'ruby-graphql', depends_on: [:tracer] do |value| + get_option(:tracer).set_service_info(value, 'ruby-graphql', Ext::AppTypes::WEB) + value + end + option :schemas, default: [] + + class << self + def patch + return patched? if patched? || !compatible? + + get_option(:schemas).each { |s| patch_schema!(s) } + + @patched = true + end + + def patch_schema!(schema) + tracer = get_option(:tracer) + service_name = get_option(:service_name) + + schema.define do + use( + ::GraphQL::Tracing::DataDogTracing, + tracer: tracer, + service: service_name + ) + end + end + + def patched? + return @patched if defined?(@patched) + @patched = false + end + + private + + def compatible? + defined?(::GraphQL) && defined?(::GraphQL::Tracing::DataDogTracing) + end + end + end + end + end +end From dd53633e6a8c42e070cbedeabc319723017442ef Mon Sep 17 00:00:00 2001 From: David Elner Date: Thu, 4 Jan 2018 18:27:36 -0500 Subject: [PATCH 02/72] Added: GraphQL tests for basic tracing. --- test/contrib/graphql/test_types.rb | 35 ++++++++++++++++++ test/contrib/graphql/tracer_test.rb | 56 +++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 test/contrib/graphql/test_types.rb create mode 100644 test/contrib/graphql/tracer_test.rb diff --git a/test/contrib/graphql/test_types.rb b/test/contrib/graphql/test_types.rb new file mode 100644 index 00000000000..71cbefef780 --- /dev/null +++ b/test/contrib/graphql/test_types.rb @@ -0,0 +1,35 @@ +module Datadog + module Contrib + module GraphQL + class Foo + attr_accessor :id, :name + + def initialize(id, name = 'bar') + @id = id + @name = name + end + end + + FooType = ::GraphQL::ObjectType.define do + name 'Foo' + field :id, !types.ID + field :name, types.String + field :created_at, !types.String + field :updated_at, !types.String + end + + QueryType = ::GraphQL::ObjectType.define do + name 'Query' + # Add root-level fields here. + # They will be entry points for queries on your schema. + + field :foo do + type FooType + argument :id, !types.ID + description 'Find a Foo by ID' + resolve ->(_obj, args, _ctx) { Foo.new(args['id']) } + end + end + end + end +end diff --git a/test/contrib/graphql/tracer_test.rb b/test/contrib/graphql/tracer_test.rb new file mode 100644 index 00000000000..3d4e491e2c4 --- /dev/null +++ b/test/contrib/graphql/tracer_test.rb @@ -0,0 +1,56 @@ +require 'helper' +require 'ddtrace' +require 'graphql' +require_relative 'test_types' + +module Datadog + module Contrib + module GraphQL + class TracerTest < Minitest::Test + def setup + @tracer = get_test_tracer + @schema = build_schema + + Datadog.configure do |c| + c.use :graphql, + service_name: 'graphql-test', + tracer: tracer, + schemas: [schema] + end + end + + def test_trace + # Perform query + query = '{ foo(id: 1) { name } }' + result = schema.execute(query, variables: {}, context: {}, operation_name: nil) + + # Expect no errors + assert_nil(result.to_h['errors']) + + # Expect nine spans + assert_equal(9, all_spans.length) + + # Expect each span to be properly named + all_spans.each do |span| + assert_equal('graphql-test', span.service) + assert_equal(true, !span.resource.to_s.empty?) + end + end + + def build_schema + ::GraphQL::Schema.define do + query(QueryType) + end + end + + private + + attr_reader :tracer, :schema + + def all_spans + tracer.writer.spans(:keep) + end + end + end + end +end From d5a5cb7ea77f14d92d6a538e5b3f9169248fbb98 Mon Sep 17 00:00:00 2001 From: David Elner Date: Thu, 1 Feb 2018 15:21:35 -0500 Subject: [PATCH 03/72] Added: GraphQL to documentation --- docs/GettingStarted.md | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index 31872664001..3feafd2d9a4 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -41,6 +41,7 @@ For further details and options, check our integrations list. * [Ruby on Rails](#Ruby_on_Rails) * [Sinatra](#Sinatra) * [Rack](#Rack) +* [GraphQL](#GraphQL) * [Grape](#Grape) * [Active Record](#Active_Record) * [Elastic Search](#Elastic_Search) @@ -144,6 +145,57 @@ Where `options` is an optional `Hash` that accepts the following parameters: ## Other libraries +### GraphQL + +*Version 1.7.9+ supported* + +The GraphQL integration activates instrumentation for GraphQL queries. To activate your integration, use the ``Datadog.configure`` method: + + # Inside Rails initializer or equivalent + Datadog.configure do |c| + c.use :graphql, + service_name: 'graphql', + schemas: [YourSchema] + end + + # Then run a GraphQL query + YourSchema.execute(query, variables: {}, context: {}, operation_name: nil) + +Where `options` is an optional `Hash` that accepts the following parameters: + +| Key | Description | Default | +| --- | --- | --- | +| ``service_name`` | Service name used for `graphql` instrumentation | ``ruby-graphql`` | +| ``schemas`` | Optional. Array of `GraphQL::Schema` objects which to trace. Tracing will be added to all the schemas listed, using the options provided to this configuration. If you supply your schema in this option, do not add `use(GraphQL::Tracing::DataDogTracing)` to the schema's definition, to avoid double traces. If you do not supply this option, you will need to individually configure your own schemas (see below.) | ``[]`` | +| ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | + +##### Manually configuring GraphQL schemas + +If you have multiple schemas, and/or you prefer to individually configure the tracer settings for each, +in the schema definition, you can add the following: + +``` +YourSchema = GraphQL::Schema.define do + use( + GraphQL::Tracing::DataDogTracing, + service: 'graphql', + tracer: Datadog.tracer + ) +end +``` + +Or you can modify an already defined schema: + +``` +YourSchema.define do + use( + GraphQL::Tracing::DataDogTracing, + service: 'graphql', + tracer: Datadog.tracer + ) +end +``` + ### Grape The Grape integration adds the instrumentation to Grape endpoints and filters. This integration can work side by side From 3dae1b68f344cd5873ccc847b97846d64b4d1638 Mon Sep 17 00:00:00 2001 From: David Elner Date: Thu, 1 Feb 2018 16:35:47 -0500 Subject: [PATCH 04/72] Fixed: GraphQL integration not auto-loading --- lib/ddtrace.rb | 1 + lib/ddtrace/contrib/graphql/patcher.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/ddtrace.rb b/lib/ddtrace.rb index 037c75d2552..664eab25d8d 100644 --- a/lib/ddtrace.rb +++ b/lib/ddtrace.rb @@ -59,6 +59,7 @@ def configure(target = configuration, opts = {}) require 'ddtrace/contrib/elasticsearch/patcher' require 'ddtrace/contrib/faraday/patcher' require 'ddtrace/contrib/grape/patcher' +require 'ddtrace/contrib/graphql/patcher' require 'ddtrace/contrib/redis/patcher' require 'ddtrace/contrib/http/patcher' require 'ddtrace/contrib/aws/patcher' diff --git a/lib/ddtrace/contrib/graphql/patcher.rb b/lib/ddtrace/contrib/graphql/patcher.rb index 2b0f0adb922..ff31087a66a 100644 --- a/lib/ddtrace/contrib/graphql/patcher.rb +++ b/lib/ddtrace/contrib/graphql/patcher.rb @@ -8,6 +8,7 @@ module GraphQL module Patcher include Base register_as :graphql + option :tracer, default: Datadog.tracer option :service_name, default: 'ruby-graphql', depends_on: [:tracer] do |value| get_option(:tracer).set_service_info(value, 'ruby-graphql', Ext::AppTypes::WEB) From 021c71cea0c554f60e4aa3808257d39d12132853 Mon Sep 17 00:00:00 2001 From: David Elner Date: Thu, 1 Feb 2018 16:37:11 -0500 Subject: [PATCH 05/72] Added: GraphQL RSpec example --- Rakefile | 4 +- spec/ddtrace/contrib/graphql/test_types.rb | 52 +++++++++++++++++++ spec/ddtrace/contrib/graphql/tracer_spec.rb | 47 +++++++++++++++++ test/contrib/graphql/test_types.rb | 35 ------------- test/contrib/graphql/tracer_test.rb | 56 --------------------- 5 files changed, 101 insertions(+), 93 deletions(-) create mode 100644 spec/ddtrace/contrib/graphql/test_types.rb create mode 100644 spec/ddtrace/contrib/graphql/tracer_spec.rb delete mode 100644 test/contrib/graphql/test_types.rb delete mode 100644 test/contrib/graphql/tracer_test.rb diff --git a/Rakefile b/Rakefile index ad6ff84893e..481fe0bcc59 100644 --- a/Rakefile +++ b/Rakefile @@ -48,6 +48,7 @@ namespace :spec do :rack, :faraday, :grape, + :graphql, :aws, :sucker_punch, :mongodb, @@ -112,7 +113,6 @@ namespace :test do :elasticsearch, :faraday, :grape, - :graphql, :http, :mongodb, :resque, @@ -213,7 +213,6 @@ task :ci do sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:sinatra' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:rack' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:grape' - sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:graphql' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:faraday' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:aws' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:mongodb' @@ -233,6 +232,7 @@ task :ci do # RSpec sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:active_record' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:dalli' + sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:graphql' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:racecar' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:dalli' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:active_record' diff --git a/spec/ddtrace/contrib/graphql/test_types.rb b/spec/ddtrace/contrib/graphql/test_types.rb new file mode 100644 index 00000000000..45053f1e005 --- /dev/null +++ b/spec/ddtrace/contrib/graphql/test_types.rb @@ -0,0 +1,52 @@ +require 'graphql' + +RSpec.shared_context 'GraphQL test schema' do + let(:schema) do + qt = query_type + + ::GraphQL::Schema.define do + query(qt) + end + end + + let(:query_type_name) { 'Query' } + let(:query_type) do + qtn = query_type_name + ot = object_type + oc = object_class + + ::GraphQL::ObjectType.define do + name qtn + field ot.name.downcase.to_sym do + type ot + argument :id, !types.ID + description 'Find an object by ID' + resolve ->(_obj, args, _ctx) { oc.new(args['id']) } + end + end + end + + let(:object_type_name) { 'Foo' } + let(:object_type) do + otn = object_type_name + + ::GraphQL::ObjectType.define do + name otn + field :id, !types.ID + field :name, types.String + field :created_at, !types.String + field :updated_at, !types.String + end + end + + let(:object_class) do + Class.new do + attr_accessor :id, :name + + def initialize(id, name = 'bar') + @id = id + @name = name + end + end + end +end diff --git a/spec/ddtrace/contrib/graphql/tracer_spec.rb b/spec/ddtrace/contrib/graphql/tracer_spec.rb new file mode 100644 index 00000000000..8c1e21eefbd --- /dev/null +++ b/spec/ddtrace/contrib/graphql/tracer_spec.rb @@ -0,0 +1,47 @@ +require 'spec_helper' +require 'ddtrace/contrib/graphql/test_types' + +require 'ddtrace' + +# rubocop:disable Metrics/BlockLength +RSpec.describe 'GraphQL patcher' do + include_context 'GraphQL test schema' + + let(:tracer) { ::Datadog::Tracer.new(writer: FauxWriter.new) } + + def all_spans + tracer.writer.spans(:keep) + end + + before(:each) do + Datadog.configure do |c| + c.use :graphql, + service_name: 'graphql-test', + tracer: tracer, + schemas: [schema] + end + end + + describe 'query trace' do + subject(:result) { schema.execute(query, variables: {}, context: {}, operation_name: nil) } + + let(:query) { '{ foo(id: 1) { name } }' } + let(:variables) { {} } + let(:context) { {} } + let(:operation_name) { nil } + + it do + # Expect no errors + expect(result.to_h['errors']).to be nil + + # Expect nine spans + expect(all_spans).to have(9).items + + # Expect each span to be properly named + all_spans.each do |span| + expect(span.service).to eq('graphql-test') + expect(span.resource.to_s).to_not be_empty + end + end + end +end diff --git a/test/contrib/graphql/test_types.rb b/test/contrib/graphql/test_types.rb deleted file mode 100644 index 71cbefef780..00000000000 --- a/test/contrib/graphql/test_types.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Datadog - module Contrib - module GraphQL - class Foo - attr_accessor :id, :name - - def initialize(id, name = 'bar') - @id = id - @name = name - end - end - - FooType = ::GraphQL::ObjectType.define do - name 'Foo' - field :id, !types.ID - field :name, types.String - field :created_at, !types.String - field :updated_at, !types.String - end - - QueryType = ::GraphQL::ObjectType.define do - name 'Query' - # Add root-level fields here. - # They will be entry points for queries on your schema. - - field :foo do - type FooType - argument :id, !types.ID - description 'Find a Foo by ID' - resolve ->(_obj, args, _ctx) { Foo.new(args['id']) } - end - end - end - end -end diff --git a/test/contrib/graphql/tracer_test.rb b/test/contrib/graphql/tracer_test.rb deleted file mode 100644 index 3d4e491e2c4..00000000000 --- a/test/contrib/graphql/tracer_test.rb +++ /dev/null @@ -1,56 +0,0 @@ -require 'helper' -require 'ddtrace' -require 'graphql' -require_relative 'test_types' - -module Datadog - module Contrib - module GraphQL - class TracerTest < Minitest::Test - def setup - @tracer = get_test_tracer - @schema = build_schema - - Datadog.configure do |c| - c.use :graphql, - service_name: 'graphql-test', - tracer: tracer, - schemas: [schema] - end - end - - def test_trace - # Perform query - query = '{ foo(id: 1) { name } }' - result = schema.execute(query, variables: {}, context: {}, operation_name: nil) - - # Expect no errors - assert_nil(result.to_h['errors']) - - # Expect nine spans - assert_equal(9, all_spans.length) - - # Expect each span to be properly named - all_spans.each do |span| - assert_equal('graphql-test', span.service) - assert_equal(true, !span.resource.to_s.empty?) - end - end - - def build_schema - ::GraphQL::Schema.define do - query(QueryType) - end - end - - private - - attr_reader :tracer, :schema - - def all_spans - tracer.writer.spans(:keep) - end - end - end - end -end From 5a53d0ff290ff4105964f8e3531c757a64d7ef85 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 5 Feb 2018 14:52:34 -0500 Subject: [PATCH 06/72] Added: Version constraint for GraphQL 1.7.9+ --- lib/ddtrace/contrib/graphql/patcher.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/ddtrace/contrib/graphql/patcher.rb b/lib/ddtrace/contrib/graphql/patcher.rb index ff31087a66a..3f8a513b827 100644 --- a/lib/ddtrace/contrib/graphql/patcher.rb +++ b/lib/ddtrace/contrib/graphql/patcher.rb @@ -46,7 +46,9 @@ def patched? private def compatible? - defined?(::GraphQL) && defined?(::GraphQL::Tracing::DataDogTracing) + defined?(::GraphQL) \ + && defined?(::GraphQL::Tracing::DataDogTracing) \ + && Gem.loaded_specs['graphql'].version >= Gem::Version.new('1.7.9') end end end From f221b2686243f6bd77a489210b28024b54c12725 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 5 Feb 2018 14:53:16 -0500 Subject: [PATCH 07/72] Added: LogHelpers to suppress warnings from graphql --- spec/ddtrace/contrib/graphql/test_types.rb | 4 +++- spec/ddtrace/contrib/graphql/tracer_spec.rb | 8 ++++++++ spec/spec_helper.rb | 2 ++ spec/support/log_helpers.rb | 15 +++++++++++++++ 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 spec/support/log_helpers.rb diff --git a/spec/ddtrace/contrib/graphql/test_types.rb b/spec/ddtrace/contrib/graphql/test_types.rb index 45053f1e005..9f674ed60ce 100644 --- a/spec/ddtrace/contrib/graphql/test_types.rb +++ b/spec/ddtrace/contrib/graphql/test_types.rb @@ -1,4 +1,6 @@ -require 'graphql' +LogHelpers.without_warnings do + require 'graphql' +end RSpec.shared_context 'GraphQL test schema' do let(:schema) do diff --git a/spec/ddtrace/contrib/graphql/tracer_spec.rb b/spec/ddtrace/contrib/graphql/tracer_spec.rb index 8c1e21eefbd..cfd38699770 100644 --- a/spec/ddtrace/contrib/graphql/tracer_spec.rb +++ b/spec/ddtrace/contrib/graphql/tracer_spec.rb @@ -7,6 +7,14 @@ RSpec.describe 'GraphQL patcher' do include_context 'GraphQL test schema' + # GraphQL generates tons of warnings. + # This suppresses those warnings. + around(:each) do |example| + without_warnings do + example.run + end + end + let(:tracer) { ::Datadog::Tracer.new(writer: FauxWriter.new) } def all_spans diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 81135016d07..d8a35131729 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -16,6 +16,7 @@ # require 'support/rails_active_record_helpers' # require 'support/configuration_helpers' require 'support/synchronization_helpers' +require 'support/log_helpers' WebMock.allow_net_connect! WebMock.disable! @@ -25,6 +26,7 @@ # config.include RailsActiveRecordHelpers # config.include ConfigurationHelpers config.include SynchronizationHelpers + config.include LogHelpers config.expect_with :rspec do |expectations| expectations.include_chain_clauses_in_custom_matcher_descriptions = true diff --git a/spec/support/log_helpers.rb b/spec/support/log_helpers.rb new file mode 100644 index 00000000000..491dffaef6d --- /dev/null +++ b/spec/support/log_helpers.rb @@ -0,0 +1,15 @@ +module LogHelpers + def without_warnings(&block) + LogHelpers.without_warnings(&block) + end + + def self.without_warnings + v = $VERBOSE + $VERBOSE = nil + begin + yield + ensure + $VERBOSE = v + end + end +end From bd10fbf5c8e3d9b477693a7984fefb101ba3b70d Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 6 Feb 2018 13:00:47 -0500 Subject: [PATCH 08/72] Changed: GraphQL schemas option and specs --- docs/GettingStarted.md | 16 +++++++-------- lib/ddtrace/contrib/graphql/patcher.rb | 4 ++-- spec/ddtrace/contrib/graphql/tracer_spec.rb | 22 +++++++++++++++++++-- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index 3feafd2d9a4..a338b62f4bf 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -161,25 +161,24 @@ The GraphQL integration activates instrumentation for GraphQL queries. To activa # Then run a GraphQL query YourSchema.execute(query, variables: {}, context: {}, operation_name: nil) -Where `options` is an optional `Hash` that accepts the following parameters: +The `use :graphql` method accepts the following parameters: | Key | Description | Default | | --- | --- | --- | | ``service_name`` | Service name used for `graphql` instrumentation | ``ruby-graphql`` | -| ``schemas`` | Optional. Array of `GraphQL::Schema` objects which to trace. Tracing will be added to all the schemas listed, using the options provided to this configuration. If you supply your schema in this option, do not add `use(GraphQL::Tracing::DataDogTracing)` to the schema's definition, to avoid double traces. If you do not supply this option, you will need to individually configure your own schemas (see below.) | ``[]`` | +| ``schemas`` | Required. Array of `GraphQL::Schema` objects which to trace. Tracing will be added to all the schemas listed, using the options provided to this configuration. If you do not provide any, then tracing will not be activated. | ``[]`` | | ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | ##### Manually configuring GraphQL schemas -If you have multiple schemas, and/or you prefer to individually configure the tracer settings for each, -in the schema definition, you can add the following: +If you prefer to individually configure the tracer settings for a schema (e.g. you have multiple schemas with different service names), +in the schema definition, you can add the following [using the GraphQL API](http://graphql-ruby.org/queries/tracing.html): ``` YourSchema = GraphQL::Schema.define do use( GraphQL::Tracing::DataDogTracing, - service: 'graphql', - tracer: Datadog.tracer + service: 'graphql' ) end ``` @@ -190,12 +189,13 @@ Or you can modify an already defined schema: YourSchema.define do use( GraphQL::Tracing::DataDogTracing, - service: 'graphql', - tracer: Datadog.tracer + service: 'graphql' ) end ``` +Do *not* `use :graphql` in `Datadog.configure` if you choose to configure manually, as to avoid double tracing. These two means of configuring GraphQL tracing are considered mutually exclusive. + ### Grape The Grape integration adds the instrumentation to Grape endpoints and filters. This integration can work side by side diff --git a/lib/ddtrace/contrib/graphql/patcher.rb b/lib/ddtrace/contrib/graphql/patcher.rb index 3f8a513b827..3425d0ed209 100644 --- a/lib/ddtrace/contrib/graphql/patcher.rb +++ b/lib/ddtrace/contrib/graphql/patcher.rb @@ -14,11 +14,11 @@ module Patcher get_option(:tracer).set_service_info(value, 'ruby-graphql', Ext::AppTypes::WEB) value end - option :schemas, default: [] + option :schemas class << self def patch - return patched? if patched? || !compatible? + return patched? if patched? || !compatible? || get_option(:schemas).nil? get_option(:schemas).each { |s| patch_schema!(s) } diff --git a/spec/ddtrace/contrib/graphql/tracer_spec.rb b/spec/ddtrace/contrib/graphql/tracer_spec.rb index cfd38699770..61811757f30 100644 --- a/spec/ddtrace/contrib/graphql/tracer_spec.rb +++ b/spec/ddtrace/contrib/graphql/tracer_spec.rb @@ -17,10 +17,13 @@ let(:tracer) { ::Datadog::Tracer.new(writer: FauxWriter.new) } - def all_spans + def pop_spans tracer.writer.spans(:keep) end + let(:all_spans) { pop_spans } + let(:root_span) { all_spans.find { |s| s.parent.nil? } } + before(:each) do Datadog.configure do |c| c.use :graphql, @@ -45,10 +48,25 @@ def all_spans # Expect nine spans expect(all_spans).to have(9).items + # List of valid resource names + # (If this is too brittle, revist later.) + valid_resource_names = [ + 'Query.foo', + 'analyze.graphql', + 'execute.graphql', + 'lex.graphql', + 'parse.graphql', + 'validate.graphql' + ] + + # Expect root span to be 'execute.graphql' + expect(root_span.name).to eq('execute.graphql') + expect(root_span.resource).to eq('execute.graphql') + # Expect each span to be properly named all_spans.each do |span| expect(span.service).to eq('graphql-test') - expect(span.resource.to_s).to_not be_empty + expect(valid_resource_names).to include(span.resource.to_s) end end end From 3e85f400db2430219aeeb0be90018efd7ea101a3 Mon Sep 17 00:00:00 2001 From: David Elner Date: Thu, 8 Feb 2018 13:42:04 -0500 Subject: [PATCH 09/72] Instrument ActiveRecord instantiations (#334) * Added: Instrumentation for Rails 4.2+ ActiveRecord instantiation * Added: Instrumentation for non-Rails ActiveRecord 4.2+ instantiation * Added: Ext::Ruby type * Changed: Rails AR instantiation span to use Rails service name and ruby type * Added: database_name option to ActiveRecord integration * Changed: How service is set for ActiveRecord spans. * Changed: ActiveRecord ruby_service_name to orm_service_name, span type to custom. * Added: instantiation_tracing_supported? to improve feature detection * Added: `orm_service_name` option to ActiveRecord documentation. * Removed: Ext::Ruby * Changed: Instrument instantiation only if supported. --- docs/GettingStarted.md | 3 +- lib/ddtrace/contrib/active_record/patcher.rb | 75 ++++++++++++++----- lib/ddtrace/contrib/rails/active_record.rb | 32 ++++++-- lib/ddtrace/contrib/rails/patcher.rb | 5 ++ .../contrib/active_record/tracer_spec.rb | 24 +++++- test/contrib/rails/controller_test.rb | 53 +++++++++---- test/contrib/rails/database_test.rb | 24 ++++++ test/contrib/rails/rack_middleware_test.rb | 23 +++++- .../sinatra/tracer_activerecord_test.rb | 32 ++++++++ 9 files changed, 225 insertions(+), 46 deletions(-) diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index a338b62f4bf..aed8a6b6784 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -248,7 +248,8 @@ Where `options` is an optional `Hash` that accepts the following parameters: | Key | Description | Default | | --- | --- | --- | -| ``service_name`` | Service name used for `active_record` instrumentation | active_record | +| ``service_name`` | Service name used for database portion of `active_record` instrumentation. | Name of database adapter (e.g. `mysql2`) | +| ``orm_service_name`` | Service name used for the Ruby ORM portion of `active_record` instrumentation. Overrides service name for ORM spans if explicitly set, which otherwise inherit their service from their parent. | ``active_record`` | ### Elastic Search diff --git a/lib/ddtrace/contrib/active_record/patcher.rb b/lib/ddtrace/contrib/active_record/patcher.rb index 4011bd499a7..c22ad470485 100644 --- a/lib/ddtrace/contrib/active_record/patcher.rb +++ b/lib/ddtrace/contrib/active_record/patcher.rb @@ -1,3 +1,6 @@ +require 'ddtrace/ext/sql' +require 'ddtrace/ext/app_types' + module Datadog module Contrib module ActiveRecord @@ -5,7 +8,10 @@ module ActiveRecord module Patcher include Base register_as :active_record, auto_patch: false - option :service_name + option :service_name do |value| + value.tap { @database_service_name = nil } + end + option :orm_service_name option :tracer, default: Datadog.tracer @patched = false @@ -21,11 +27,8 @@ def patch if !@patched && defined?(::ActiveRecord) begin require 'ddtrace/contrib/rails/utils' - require 'ddtrace/ext/sql' - require 'ddtrace/ext/app_types' - - patch_active_record() + patch_active_record @patched = true rescue StandardError => e Datadog::Tracer.log.error("Unable to apply Active Record integration: #{e}") @@ -40,6 +43,27 @@ def patch_active_record ::ActiveSupport::Notifications.subscribe('sql.active_record') do |*args| sql(*args) end + + if instantiation_tracing_supported? + # subscribe when the active record instantiates objects + ::ActiveSupport::Notifications.subscribe('instantiation.active_record') do |*args| + instantiation(*args) + end + end + end + + def instantiation_tracing_supported? + Gem.loaded_specs['activerecord'] \ + && Gem.loaded_specs['activerecord'].version >= Gem::Version.new('4.2') + end + + # NOTE: Resolve this here instead of in the option defaults, + # because resolving adapter name as a default causes ActiveRecord to connect, + # which isn't a good idea at initialization time. + def self.database_service_name + @database_service_name ||= (get_option(:service_name) || adapter_name).tap do |name| + get_option(:tracer).set_service_info(name, 'active_record', Ext::AppTypes::DB) + end end def self.adapter_name @@ -58,22 +82,12 @@ def self.adapter_port @adapter_port ||= Datadog::Contrib::Rails::Utils.adapter_port end - def self.database_service - return @database_service if defined?(@database_service) - - @database_service = get_option(:service_name) || adapter_name - get_option(:tracer).set_service_info(@database_service, 'active_record', Ext::AppTypes::DB) - @database_service - end - def self.sql(_name, start, finish, _id, payload) - span_type = Datadog::Ext::SQL::TYPE - span = get_option(:tracer).trace( "#{adapter_name}.query", resource: payload.fetch(:sql), - service: database_service, - span_type: span_type + service: database_service_name, + span_type: Datadog::Ext::SQL::TYPE ) # Find out if the SQL query has been cached in this request. This meta is really @@ -84,7 +98,6 @@ def self.sql(_name, start, finish, _id, payload) # the span should have the query ONLY in the Resource attribute, # so that the ``sql.query`` tag will be set in the agent with an # obfuscated version - span.span_type = Datadog::Ext::SQL::TYPE span.set_tag('active_record.db.vendor', adapter_name) span.set_tag('active_record.db.name', database_name) span.set_tag('active_record.db.cached', cached) if cached @@ -93,7 +106,31 @@ def self.sql(_name, start, finish, _id, payload) span.start_time = start span.finish(finish) rescue StandardError => e - Datadog::Tracer.log.error(e.message) + Datadog::Tracer.log.debug(e.message) + end + + def self.instantiation(_name, start, finish, _id, payload) + span = get_option(:tracer).trace( + 'active_record.instantiation', + resource: payload.fetch(:class_name), + span_type: 'custom' + ) + + # Inherit service name from parent, if available. + span.service = if get_option(:orm_service_name) + get_option(:orm_service_name) + elsif span.parent + span.parent.service + else + 'active_record' + end + + span.set_tag('active_record.instantiation.class_name', payload.fetch(:class_name)) + span.set_tag('active_record.instantiation.record_count', payload.fetch(:record_count)) + span.start_time = start + span.finish(finish) + rescue StandardError => e + Datadog::Tracer.log.debug(e.message) end end end diff --git a/lib/ddtrace/contrib/rails/active_record.rb b/lib/ddtrace/contrib/rails/active_record.rb index b8c2af1053e..f7ee76cc1c9 100644 --- a/lib/ddtrace/contrib/rails/active_record.rb +++ b/lib/ddtrace/contrib/rails/active_record.rb @@ -1,5 +1,4 @@ require 'ddtrace/ext/sql' - require 'ddtrace/contrib/rails/utils' module Datadog @@ -15,6 +14,13 @@ def self.instrument ::ActiveSupport::Notifications.subscribe('sql.active_record') do |*args| sql(*args) end + + if Datadog::Contrib::Rails::Patcher.active_record_instantiation_tracing_supported? + # subscribe when the active record instantiates objects + ::ActiveSupport::Notifications.subscribe('instantiation.active_record') do |*args| + instantiation(*args) + end + end end def self.sql(_name, start, finish, _id, payload) @@ -24,13 +30,12 @@ def self.sql(_name, start, finish, _id, payload) database_name = Datadog::Contrib::Rails::Utils.database_name adapter_host = Datadog::Contrib::Rails::Utils.adapter_host adapter_port = Datadog::Contrib::Rails::Utils.adapter_port - span_type = Datadog::Ext::SQL::TYPE span = tracer.trace( "#{adapter_name}.query", resource: payload.fetch(:sql), service: database_service, - span_type: span_type + span_type: Datadog::Ext::SQL::TYPE ) # Find out if the SQL query has been cached in this request. This meta is really @@ -41,7 +46,6 @@ def self.sql(_name, start, finish, _id, payload) # the span should have the query ONLY in the Resource attribute, # so that the ``sql.query`` tag will be set in the agent with an # obfuscated version - span.span_type = Datadog::Ext::SQL::TYPE span.set_tag('rails.db.vendor', adapter_name) span.set_tag('rails.db.name', database_name) span.set_tag('rails.db.cached', cached) if cached @@ -50,7 +54,25 @@ def self.sql(_name, start, finish, _id, payload) span.start_time = start span.finish(finish) rescue StandardError => e - Datadog::Tracer.log.error(e.message) + Datadog::Tracer.log.debug(e.message) + end + + def self.instantiation(_name, start, finish, _id, payload) + tracer = Datadog.configuration[:rails][:tracer] + + span = tracer.trace( + 'active_record.instantiation', + resource: payload.fetch(:class_name), + span_type: 'custom' + ) + + span.service = span.parent ? span.parent.service : Datadog.configuration[:rails][:service_name] + span.set_tag('active_record.instantiation.class_name', payload.fetch(:class_name)) + span.set_tag('active_record.instantiation.record_count', payload.fetch(:record_count)) + span.start_time = start + span.finish(finish) + rescue StandardError => e + Datadog::Tracer.log.debug(e.message) end end end diff --git a/lib/ddtrace/contrib/rails/patcher.rb b/lib/ddtrace/contrib/rails/patcher.rb index 12a8030d86e..4f943b0f7d9 100644 --- a/lib/ddtrace/contrib/rails/patcher.rb +++ b/lib/ddtrace/contrib/rails/patcher.rb @@ -36,6 +36,11 @@ def compatible? defined?(::Rails::VERSION) && ::Rails::VERSION::MAJOR.to_i >= 3 end + + def active_record_instantiation_tracing_supported? + Gem.loaded_specs['activerecord'] \ + && Gem.loaded_specs['activerecord'].version >= Gem::Version.new('4.2') + end end end end diff --git a/spec/ddtrace/contrib/active_record/tracer_spec.rb b/spec/ddtrace/contrib/active_record/tracer_spec.rb index 357b8587cdb..3a92fad4273 100644 --- a/spec/ddtrace/contrib/active_record/tracer_spec.rb +++ b/spec/ddtrace/contrib/active_record/tracer_spec.rb @@ -5,10 +5,11 @@ RSpec.describe 'ActiveRecord instrumentation' do let(:tracer) { ::Datadog::Tracer.new(writer: FauxWriter.new) } + let(:configuration_options) { { tracer: tracer } } before(:each) do Datadog.configure do |c| - c.use :active_record, tracer: tracer + c.use :active_record, configuration_options end end @@ -33,4 +34,25 @@ expect(span.get_tag('out.port')).to eq('53306') expect(span.get_tag('sql.query')).to eq(nil) end + + context 'when service_name' do + subject(:spans) do + Article.count + tracer.writer.spans + end + + let(:query_span) { spans.first } + + context 'is not set' do + let(:configuration_options) { super().merge({ service_name: nil }) } + it { expect(query_span.service).to eq('mysql2') } + end + + context 'is set' do + let(:service_name) { 'test_active_record' } + let(:configuration_options) { super().merge({ service_name: service_name }) } + + it { expect(query_span.service).to eq(service_name) } + end + end end diff --git a/test/contrib/rails/controller_test.rb b/test/contrib/rails/controller_test.rb index aacf464747b..1473c4d9d67 100644 --- a/test/contrib/rails/controller_test.rb +++ b/test/contrib/rails/controller_test.rb @@ -114,23 +114,44 @@ class TracingControllerTest < ActionController::TestCase # render the endpoint get :full assert_response :success - spans = @tracer.writer.spans() - assert_equal(spans.length, 4) - - span_database, span_request, span_cache, span_template = spans - - # assert the spans - adapter_name = get_adapter_name() - assert_equal(span_cache.name, 'rails.cache') - assert_equal(span_database.name, "#{adapter_name}.query") - assert_equal(span_template.name, 'rails.render_template') - assert_equal(span_request.name, 'rails.action_controller') + spans = @tracer.writer.spans - # assert the parenting - assert_nil(span_request.parent) - assert_equal(span_template.parent, span_request) - assert_equal(span_database.parent, span_template) - assert_equal(span_cache.parent, span_request) + # rubocop:disable Style/IdenticalConditionalBranches + if Datadog::Contrib::Rails::Patcher.active_record_instantiation_tracing_supported? + assert_equal(spans.length, 5) + span_instantiation, span_database, span_request, span_cache, span_template = spans + + # assert the spans + adapter_name = get_adapter_name + assert_equal(span_instantiation.name, 'active_record.instantiation') + assert_equal(span_cache.name, 'rails.cache') + assert_equal(span_database.name, "#{adapter_name}.query") + assert_equal(span_template.name, 'rails.render_template') + assert_equal(span_request.name, 'rails.action_controller') + + # assert the parenting + assert_nil(span_request.parent) + assert_equal(span_template.parent, span_request) + assert_equal(span_database.parent, span_template) + assert_equal(span_instantiation.parent, span_template) + assert_equal(span_cache.parent, span_request) + else + assert_equal(spans.length, 4) + span_database, span_request, span_cache, span_template = spans + + # assert the spans + adapter_name = get_adapter_name + assert_equal(span_cache.name, 'rails.cache') + assert_equal(span_database.name, "#{adapter_name}.query") + assert_equal(span_template.name, 'rails.render_template') + assert_equal(span_request.name, 'rails.action_controller') + + # assert the parenting + assert_nil(span_request.parent) + assert_equal(span_template.parent, span_request) + assert_equal(span_database.parent, span_template) + assert_equal(span_cache.parent, span_request) + end end test 'multiple calls should not leave an unfinished span in the local thread buffer' do diff --git a/test/contrib/rails/database_test.rb b/test/contrib/rails/database_test.rb index 9524756e0a3..fb84318283c 100644 --- a/test/contrib/rails/database_test.rb +++ b/test/contrib/rails/database_test.rb @@ -37,6 +37,30 @@ class DatabaseTracingTest < ActiveSupport::TestCase assert_nil(span.get_tag('sql.query')) end + test 'active record traces instantiation' do + if Datadog::Contrib::Rails::Patcher.active_record_instantiation_tracing_supported? + begin + Article.create(title: 'Instantiation test') + @tracer.writer.spans # Clear spans + + # make the query and assert the proper spans + Article.all.entries + spans = @tracer.writer.spans + assert_equal(2, spans.length) + + instantiation_span = spans.first + assert_equal(instantiation_span.name, 'active_record.instantiation') + assert_equal(instantiation_span.span_type, 'custom') + assert_equal(instantiation_span.service, Datadog.configuration[:rails][:service_name]) + assert_equal(instantiation_span.resource, 'Article') + assert_equal(instantiation_span.get_tag('active_record.instantiation.class_name'), 'Article') + assert_equal(instantiation_span.get_tag('active_record.instantiation.record_count'), '1') + ensure + Article.delete_all + end + end + end + test 'active record is sets cached tag' do # Make sure query caching is enabled... Article.cache do diff --git a/test/contrib/rails/rack_middleware_test.rb b/test/contrib/rails/rack_middleware_test.rb index 053f2b963ea..e54c3b71594 100644 --- a/test/contrib/rails/rack_middleware_test.rb +++ b/test/contrib/rails/rack_middleware_test.rb @@ -25,19 +25,25 @@ class FullStackTest < ActionDispatch::IntegrationTest Rails.application.app.instance_variable_set(:@tracer, @rack_tracer) end + # rubocop:disable Metrics/BlockLength test 'a full request is properly traced' do # make the request and assert the proper span get '/full' assert_response :success # get spans - spans = @tracer.writer.spans() - assert_equal(spans.length, 5) + spans = @tracer.writer.spans # spans are sorted alphabetically, and ... controller names start # either by m or p (MySQL or PostGreSQL) so the database span is always # the first one. Would fail with an adapter named z-something. - database_span, request_span, controller_span, cache_span, render_span = spans + if Datadog::Contrib::Rails::Patcher.active_record_instantiation_tracing_supported? + assert_equal(spans.length, 6) + instantiation_span, database_span, request_span, controller_span, cache_span, render_span = spans + else + assert_equal(spans.length, 5) + database_span, request_span, controller_span, cache_span, render_span = spans + end assert_equal(request_span.name, 'rack.request') assert_equal(request_span.span_type, 'http') @@ -57,7 +63,7 @@ class FullStackTest < ActionDispatch::IntegrationTest assert_equal(render_span.resource, 'rails.render_template') assert_equal(render_span.get_tag('rails.template_name'), 'tracing/full.html.erb') - adapter_name = get_adapter_name() + adapter_name = get_adapter_name assert_equal(database_span.name, "#{adapter_name}.query") assert_equal(database_span.span_type, 'sql') assert_equal(database_span.service, adapter_name) @@ -67,6 +73,15 @@ class FullStackTest < ActionDispatch::IntegrationTest assert_includes(database_span.resource, 'FROM') assert_includes(database_span.resource, 'articles') + if Datadog::Contrib::Rails::Patcher.active_record_instantiation_tracing_supported? + assert_equal(instantiation_span.name, 'active_record.instantiation') + assert_equal(instantiation_span.span_type, 'custom') + assert_equal(instantiation_span.service, Datadog.configuration[:rails][:service_name]) + assert_equal(instantiation_span.resource, 'Article') + assert_equal(instantiation_span.get_tag('active_record.instantiation.class_name'), 'Article') + assert_equal(instantiation_span.get_tag('active_record.instantiation.record_count'), '0') + end + assert_equal(cache_span.name, 'rails.cache') assert_equal(cache_span.span_type, 'cache') assert_equal(cache_span.resource, 'SET') diff --git a/test/contrib/sinatra/tracer_activerecord_test.rb b/test/contrib/sinatra/tracer_activerecord_test.rb index a1646d60c61..79db4daad96 100644 --- a/test/contrib/sinatra/tracer_activerecord_test.rb +++ b/test/contrib/sinatra/tracer_activerecord_test.rb @@ -25,6 +25,10 @@ class TracerActiveRecordTestApp < Sinatra::Application Article.count end end + + get '/select_request' do + Article.all.entries.to_s + end end def app @@ -114,6 +118,34 @@ def test_cached_tag assert_equal('true', spans.last.get_tag('active_record.db.cached')) end + def test_instantiation_tracing + # Only supported in Rails 4.2+ + skip unless Datadog::Contrib::ActiveRecord::Patcher.instantiation_tracing_supported? + + # Make sure Article table exists + migrate_db + Article.create(title: 'Instantiation test') + @writer.spans # Clear spans + + # Run query + get '/select_request' + assert_equal(200, last_response.status) + + spans = @writer.spans + assert_equal(3, spans.length) + + instantiation_span, sinatra_span, sqlite_span = spans + + assert_equal(instantiation_span.name, 'active_record.instantiation') + assert_equal(instantiation_span.span_type, 'custom') + assert_equal(instantiation_span.service, sinatra_span.service) + assert_equal(instantiation_span.resource, 'TracerActiveRecordTest::Article') + assert_equal(instantiation_span.get_tag('active_record.instantiation.class_name'), 'TracerActiveRecordTest::Article') + assert_equal(instantiation_span.get_tag('active_record.instantiation.record_count'), '1') + assert_equal(sinatra_span, instantiation_span.parent) + assert_equal(sinatra_span, sqlite_span.parent) + end + private def all_spans From 2c940b022cc66d64d48f8f419ecfe1db2dd345ac Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 7 Feb 2018 16:09:49 -0500 Subject: [PATCH 10/72] bumping version 0.11.2 => 0.12.0.beta1 --- lib/ddtrace/version.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/ddtrace/version.rb b/lib/ddtrace/version.rb index 3c43f23e42b..b90db2c38ad 100644 --- a/lib/ddtrace/version.rb +++ b/lib/ddtrace/version.rb @@ -1,9 +1,10 @@ module Datadog module VERSION MAJOR = 0 - MINOR = 11 - PATCH = 2 + MINOR = 12 + PATCH = 0 + PRE = 'beta1'.freeze - STRING = [MAJOR, MINOR, PATCH].compact.join('.') + STRING = [MAJOR, MINOR, PATCH, PRE].compact.join('.') end end From 3ebb56fff446d17f529b3898568a69129bd63b15 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 9 Feb 2018 14:43:15 -0500 Subject: [PATCH 11/72] Added: Faraday middleware spec --- .../contrib/faraday/middleware_spec.rb | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 spec/ddtrace/contrib/faraday/middleware_spec.rb diff --git a/spec/ddtrace/contrib/faraday/middleware_spec.rb b/spec/ddtrace/contrib/faraday/middleware_spec.rb new file mode 100644 index 00000000000..a0bd699446b --- /dev/null +++ b/spec/ddtrace/contrib/faraday/middleware_spec.rb @@ -0,0 +1,170 @@ +require 'spec_helper' + +require 'ddtrace' +require 'faraday' +require 'ddtrace/ext/distributed' + +RSpec.describe 'Faraday middleware' do + let(:tracer) { Datadog::Tracer.new(writer: FauxWriter.new) } + + let(:client) do + ::Faraday.new('http://example.com') do |builder| + builder.use(:ddtrace, middleware_options) + builder.adapter(:test) do |stub| + stub.get('/success') { |_| [200, {}, 'OK'] } + stub.post('/failure') { |_| [500, {}, 'Boom!'] } + stub.get('/not_found') { |_| [404, {}, 'Not Found.'] } + end + end + end + + let(:middleware_options) { {} } + let(:configuration_options) { { tracer: tracer } } + + let(:request_span) do + tracer.writer.spans(:keep).find { |span| span.name == Datadog::Contrib::Faraday::NAME } + end + + before(:each) do + Datadog.configure do |c| + c.use :faraday, configuration_options + end + + # Have to manually update this because its still + # using global pin instead of configuration. + # Remove this when we remove the pin. + Datadog::Pin.get_from(::Faraday).tracer = tracer + end + + context 'when there is no interference' do + subject!(:response) { client.get('/success') } + + it do + expect(response).to be_a_kind_of(::Faraday::Response) + expect(response.body).to eq('OK') + expect(response.status).to eq(200) + end + end + + context 'when there is successful request' do + subject!(:response) { client.get('/success') } + + it do + expect(request_span).to_not be nil + expect(request_span.service).to eq(Datadog::Contrib::Faraday::SERVICE) + expect(request_span.name).to eq(Datadog::Contrib::Faraday::NAME) + expect(request_span.resource).to eq('GET') + expect(request_span.get_tag(Datadog::Ext::HTTP::METHOD)).to eq('GET') + expect(request_span.get_tag(Datadog::Ext::HTTP::STATUS_CODE)).to eq('200') + expect(request_span.get_tag(Datadog::Ext::HTTP::URL)).to eq('/success') + expect(request_span.get_tag(Datadog::Ext::NET::TARGET_HOST)).to eq('example.com') + expect(request_span.get_tag(Datadog::Ext::NET::TARGET_PORT)).to eq('80') + expect(request_span.span_type).to eq(Datadog::Ext::HTTP::TYPE) + expect(request_span.status).to_not eq(Datadog::Ext::Errors::STATUS) + end + end + + context 'when there is a failing request' do + subject!(:response) { client.post('/failure') } + + it do + expect(request_span.service).to eq(Datadog::Contrib::Faraday::SERVICE) + expect(request_span.name).to eq(Datadog::Contrib::Faraday::NAME) + expect(request_span.resource).to eq('POST') + expect(request_span.get_tag(Datadog::Ext::HTTP::METHOD)).to eq('POST') + expect(request_span.get_tag(Datadog::Ext::HTTP::URL)).to eq('/failure') + expect(request_span.get_tag(Datadog::Ext::HTTP::STATUS_CODE)).to eq('500') + expect(request_span.get_tag(Datadog::Ext::NET::TARGET_HOST)).to eq('example.com') + expect(request_span.get_tag(Datadog::Ext::NET::TARGET_PORT)).to eq('80') + expect(request_span.span_type).to eq(Datadog::Ext::HTTP::TYPE) + expect(request_span.status).to eq(Datadog::Ext::Errors::STATUS) + expect(request_span.get_tag(Datadog::Ext::Errors::TYPE)).to eq('Error 500') + expect(request_span.get_tag(Datadog::Ext::Errors::MSG)).to eq('Boom!') + end + end + + context 'when there is a client error' do + subject!(:response) { client.get('/not_found') } + + it { expect(request_span.status).to_not eq(Datadog::Ext::Errors::STATUS) } + end + + context 'when there is custom error handling' do + subject!(:response) { client.get('not_found') } + + let(:middleware_options) { { error_handler: custom_handler } } + let(:custom_handler) { ->(env) { (400...600).cover?(env[:status]) } } + it { expect(request_span.status).to eq(Datadog::Ext::Errors::STATUS) } + end + + context 'when split by domain' do + subject!(:response) { client.get('/success') } + + let(:middleware_options) { { split_by_domain: true } } + + it do + expect(request_span.name).to eq(Datadog::Contrib::Faraday::NAME) + expect(request_span.service).to eq('example.com') + expect(request_span.resource).to eq('GET') + end + end + + context 'default request headers' do + subject(:response) { client.get('/success') } + + let(:headers) { response.env.request_headers } + + it do + expect(headers).to_not include(Datadog::Ext::DistributedTracing::HTTP_HEADER_TRACE_ID) + expect(headers).to_not include(Datadog::Ext::DistributedTracing::HTTP_HEADER_PARENT_ID) + end + end + + context 'when distributed tracing is enabled' do + subject(:response) { client.get('/success') } + + let(:middleware_options) { { distributed_tracing: true } } + let(:headers) { response.env.request_headers } + + it do + expect(headers[Datadog::Ext::DistributedTracing::HTTP_HEADER_TRACE_ID]).to eq(request_span.trace_id.to_s) + expect(headers[Datadog::Ext::DistributedTracing::HTTP_HEADER_PARENT_ID]).to eq(request_span.span_id.to_s) + end + + context 'but the tracer is disabled' do + before(:each) { tracer.enabled = false } + it do + expect(headers).to_not include(Datadog::Ext::DistributedTracing::HTTP_HEADER_TRACE_ID) + expect(headers).to_not include(Datadog::Ext::DistributedTracing::HTTP_HEADER_PARENT_ID) + expect(request_span).to be nil + end + end + end + + context 'global service name' do + let(:service_name) { 'faraday-global' } + + before(:each) do + @old_service_name = Datadog.configuration[:faraday][:service_name] + Datadog.configure { |c| c.use :faraday, service_name: service_name } + end + + after(:each) { Datadog.configure { |c| c.use :faraday, service_name: @old_service_name } } + + it do + client.get('/success') + expect(request_span.service).to eq(service_name) + end + end + + context 'service name per request' do + subject!(:response) { client.get('/success') } + + let(:middleware_options) { { service_name: service_name } } + let(:service_name) { 'adhoc-request' } + + it do + expect(request_span.service).to eq(service_name) + end + end +end From da51747970344756ed19b943c1b2e7b682b9a4e1 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 9 Feb 2018 14:47:44 -0500 Subject: [PATCH 12/72] Removed: Faraday minitest in favor of spec. --- Rakefile | 7 +- test/contrib/faraday/middleware_test.rb | 162 ------------------------ 2 files changed, 3 insertions(+), 166 deletions(-) delete mode 100644 test/contrib/faraday/middleware_test.rb diff --git a/Rakefile b/Rakefile index 481fe0bcc59..024eb80e85c 100644 --- a/Rakefile +++ b/Rakefile @@ -111,7 +111,6 @@ namespace :test do [ :aws, :elasticsearch, - :faraday, :grape, :http, :mongodb, @@ -213,7 +212,6 @@ task :ci do sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:sinatra' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:rack' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:grape' - sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:faraday' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:aws' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:mongodb' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:sucker_punch' @@ -224,7 +222,6 @@ task :ci do sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:redis' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:sinatra' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:rack' - sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:faraday' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:aws' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:mongodb' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:sucker_punch' @@ -232,10 +229,12 @@ task :ci do # RSpec sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:active_record' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:dalli' + sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:faraday' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:graphql' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:racecar' - sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:dalli' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:active_record' + sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:dalli' + sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:faraday' when 2 sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:sidekiq' sh 'rvm $SIDEKIQ_OLD_VERSIONS --verbose do appraisal contrib-old rake test:sidekiq' diff --git a/test/contrib/faraday/middleware_test.rb b/test/contrib/faraday/middleware_test.rb deleted file mode 100644 index ce72475bf06..00000000000 --- a/test/contrib/faraday/middleware_test.rb +++ /dev/null @@ -1,162 +0,0 @@ -require 'helper' -require 'ddtrace' -require 'faraday' -require 'ddtrace/ext/distributed' - -module Datadog - module Contrib - module Faraday - class MiddlewareTest < Minitest::Test - def setup - Datadog.configure do |c| - c.use :faraday - end - - ::Faraday.datadog_pin.tracer = get_test_tracer - end - - def teardown - Datadog.configuration[:faraday].reset_options! - end - - def test_no_interference - response = client.get('/success') - - assert_kind_of(::Faraday::Response, response) - assert_equal(response.body, 'OK') - assert_equal(response.status, 200) - end - - def test_successful_request - client.get('/success') - span = request_span - - assert_equal(SERVICE, span.service) - assert_equal(NAME, span.name) - assert_equal('GET', span.resource) - assert_equal('GET', span.get_tag(Ext::HTTP::METHOD)) - assert_equal('200', span.get_tag(Ext::HTTP::STATUS_CODE)) - assert_equal('/success', span.get_tag(Ext::HTTP::URL)) - assert_equal('example.com', span.get_tag(Ext::NET::TARGET_HOST)) - assert_equal('80', span.get_tag(Ext::NET::TARGET_PORT)) - assert_equal(Ext::HTTP::TYPE, span.span_type) - refute_equal(Ext::Errors::STATUS, span.status) - end - - def test_error_response - client.post('/failure') - span = request_span - - assert_equal(SERVICE, span.service) - assert_equal(NAME, span.name) - assert_equal('POST', span.resource) - assert_equal('POST', span.get_tag(Ext::HTTP::METHOD)) - assert_equal('/failure', span.get_tag(Ext::HTTP::URL)) - assert_equal('500', span.get_tag(Ext::HTTP::STATUS_CODE)) - assert_equal('example.com', span.get_tag(Ext::NET::TARGET_HOST)) - assert_equal('80', span.get_tag(Ext::NET::TARGET_PORT)) - assert_equal(Ext::HTTP::TYPE, span.span_type) - assert_equal(Ext::Errors::STATUS, span.status) - assert_equal('Error 500', span.get_tag(Ext::Errors::TYPE)) - assert_equal('Boom!', span.get_tag(Ext::Errors::MSG)) - end - - def test_client_error - client.get('/not_found') - span = request_span - - refute_equal(Ext::Errors::STATUS, span.status) - end - - def test_custom_error_handling - custom_handler = ->(env) { (400...600).cover?(env[:status]) } - client(error_handler: custom_handler).get('not_found') - span = request_span - - assert_equal(Ext::Errors::STATUS, span.status) - end - - def test_split_by_domain_option - client(split_by_domain: true).get('/success') - span = request_span - - assert_equal(span.name, NAME) - assert_equal(span.service, 'example.com') - assert_equal(span.resource, 'GET') - end - - def test_default_tracing_headers - response = client.get('/success') - headers = response.env.request_headers - - refute_includes(headers, Ext::DistributedTracing::HTTP_HEADER_TRACE_ID) - refute_includes(headers, Ext::DistributedTracing::HTTP_HEADER_PARENT_ID) - end - - def test_distributed_tracing - response = client(distributed_tracing: true).get('/success') - headers = response.env.request_headers - span = request_span - - assert_equal(headers[Ext::DistributedTracing::HTTP_HEADER_TRACE_ID], span.trace_id.to_s) - assert_equal(headers[Ext::DistributedTracing::HTTP_HEADER_PARENT_ID], span.span_id.to_s) - end - - def test_distributed_tracing_disabled - tracer.enabled = false - - response = client(distributed_tracing: true).get('/success') - headers = response.env.request_headers - span = request_span - - # headers should not be set when the tracer is disabled: we do not want the callee - # to refer to spans which will never be sent to the agent. - assert_nil(headers[Ext::DistributedTracing::HTTP_HEADER_TRACE_ID]) - assert_nil(headers[Ext::DistributedTracing::HTTP_HEADER_PARENT_ID]) - assert_nil(span, 'disabled tracer, no spans should reach the writer') - - tracer.enabled = true - end - - def test_global_service_name - Datadog.configure do |c| - c.use :faraday, service_name: 'faraday-global' - end - - client.get('/success') - span = request_span - assert_equal('faraday-global', span.service) - end - - def test_per_request_service_name - client(service_name: 'adhoc-request').get('/success') - span = request_span - assert_equal('adhoc-request', span.service) - end - - private - - attr_reader :client - - def client(options = {}) - ::Faraday.new('http://example.com') do |builder| - builder.use(:ddtrace, options) - builder.adapter(:test) do |stub| - stub.get('/success') { |_| [200, {}, 'OK'] } - stub.post('/failure') { |_| [500, {}, 'Boom!'] } - stub.get('/not_found') { |_| [404, {}, 'Not Found.'] } - end - end - end - - def request_span - tracer.writer.spans.find { |span| span.name == NAME } - end - - def tracer - ::Faraday.datadog_pin.tracer - end - end - end - end -end From 325d03c27b2b73b50c386905e7039913c47208cf Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 9 Feb 2018 18:20:27 -0500 Subject: [PATCH 13/72] Added: Redis specs. --- .../ddtrace/contrib/redis/integration_spec.rb | 37 ++++ .../contrib/redis/method_replaced_spec.rb | 29 +++ spec/ddtrace/contrib/redis/miniapp_spec.rb | 101 +++++++++ spec/ddtrace/contrib/redis/quantize_spec.rb | 72 ++++++ spec/ddtrace/contrib/redis/redis_spec.rb | 208 ++++++++++++++++++ 5 files changed, 447 insertions(+) create mode 100644 spec/ddtrace/contrib/redis/integration_spec.rb create mode 100644 spec/ddtrace/contrib/redis/method_replaced_spec.rb create mode 100644 spec/ddtrace/contrib/redis/miniapp_spec.rb create mode 100644 spec/ddtrace/contrib/redis/quantize_spec.rb create mode 100644 spec/ddtrace/contrib/redis/redis_spec.rb diff --git a/spec/ddtrace/contrib/redis/integration_spec.rb b/spec/ddtrace/contrib/redis/integration_spec.rb new file mode 100644 index 00000000000..3d20ad66312 --- /dev/null +++ b/spec/ddtrace/contrib/redis/integration_spec.rb @@ -0,0 +1,37 @@ +require 'spec_helper' + +require 'time' +require 'redis' +require 'hiredis' +require 'ddtrace' + +RSpec.describe 'Redis integration test' do + # Use real tracer + let(:tracer) { Datadog::Tracer.new } + + before(:each) do + skip unless ENV['TEST_DATADOG_INTEGRATION'] + + # Make sure to reset default tracer + Datadog.configure do |c| + c.use :redis, tracer: tracer + end + end + + after(:each) do + Datadog.configure do |c| + c.use :redis + end + end + + let(:redis) { Redis.new(host: host, port: port) } + let(:host) { '127.0.0.1' } + let(:port) { 46379 } + + it do + expect(redis.set 'FOO', 'bar').to eq('OK') + expect(redis.get 'FOO').to eq('bar') + try_wait_until(attempts: 30) { tracer.writer.stats[:traces_flushed] >= 2 } + expect(tracer.writer.stats[:traces_flushed]).to be >= 2 + end +end diff --git a/spec/ddtrace/contrib/redis/method_replaced_spec.rb b/spec/ddtrace/contrib/redis/method_replaced_spec.rb new file mode 100644 index 00000000000..1336fba172e --- /dev/null +++ b/spec/ddtrace/contrib/redis/method_replaced_spec.rb @@ -0,0 +1,29 @@ +require 'spec_helper' + +require 'redis' +require 'hiredis' +require 'ddtrace' + +RSpec.describe 'Redis replace method test' do + before(:each) do + skip unless ENV['TEST_DATADOG_INTEGRATION'] + + Datadog.configure do |c| + c.use :redis + end + end + + let(:redis) { Redis.new(host: host, port: port) } + let(:host) { '127.0.0.1' } + let(:port) { 46379 } + + let(:call_without_datadog_method) do + Redis::Client.instance_methods.find { |m| m == :call_without_datadog } + end + + it do + expect(call_without_datadog_method).to_not be nil + expect(redis).to receive(:call).once.and_call_original + redis.call(['ping', 'hello world']) + end +end diff --git a/spec/ddtrace/contrib/redis/miniapp_spec.rb b/spec/ddtrace/contrib/redis/miniapp_spec.rb new file mode 100644 index 00000000000..34606e457f2 --- /dev/null +++ b/spec/ddtrace/contrib/redis/miniapp_spec.rb @@ -0,0 +1,101 @@ +require 'spec_helper' + +require 'time' +require 'redis' +require 'hiredis' +require 'ddtrace' + +RSpec.describe 'Redis mini app test' do + let(:tracer) { ::Datadog::Tracer.new(writer: FauxWriter.new) } + + def all_spans + tracer.writer.spans(:keep) + end + + before(:each) do + # Patch redis (don't bother configuring tracer) + Datadog.configure { |c| c.use :redis } + + # Configure client instance with tracer + Datadog.configure(client, tracer: tracer) + end + + let(:client) do + if Gem::Version.new(::Redis::VERSION) >= Gem::Version.new('4.0.0') + redis._client + else + redis.client + end + end + + let(:redis) { Redis.new(host: host, port: port) } + let(:host) { '127.0.0.1' } + let(:port) { 46379 } + + context 'when a trace is performed' do + before(:each) do + # now this is how you make sure that the redis spans are sub-spans + # of the apps parent spans: + tracer.trace('publish') do |span| + span.service = 'webapp' + span.resource = '/index' + tracer.trace('process') do |subspan| + subspan.service = 'datalayer' + subspan.resource = 'home' + redis.get 'data1' + redis.pipelined do + redis.set 'data2', 'something' + redis.get 'data2' + end + end + end + end + + # span[1] (publish_span) + # \ + # ------> span[0] (process_span) + # \ + # |-----> span[2] (redis_cmd1_span) + # \-----> span[3] (redis_cmd2_span) + let(:publish_span) { all_spans[1] } + let(:process_span) { all_spans[0] } + let(:redis_cmd1_span) { all_spans[2] } + let(:redis_cmd2_span) { all_spans[3] } + + it { expect(all_spans).to have(4).items } + + describe '"publish span"' do + it do + expect(publish_span.name).to eq('publish') + expect(publish_span.service).to eq('webapp') + expect(publish_span.resource).to eq('/index') + expect(publish_span.span_id).to_not eq(publish_span.trace_id) + expect(publish_span.parent_id).to eq(0) + end + end + + describe '"process span"' do + it do + expect(process_span.name).to eq('process') + expect(process_span.service).to eq('datalayer') + expect(process_span.resource).to eq('home') + expect(process_span.parent_id).to eq(publish_span.span_id) + expect(process_span.trace_id).to eq(publish_span.trace_id) + end + end + + describe '"command spans"' do + it do + expect(redis_cmd1_span.name).to eq('redis.command') + expect(redis_cmd1_span.service).to eq('redis') + expect(redis_cmd1_span.parent_id).to eq(process_span.span_id) + expect(redis_cmd1_span.trace_id).to eq(publish_span.trace_id) + + expect(redis_cmd2_span.name).to eq('redis.command') + expect(redis_cmd2_span.service).to eq('redis') + expect(redis_cmd2_span.parent_id).to eq(process_span.span_id) + expect(redis_cmd2_span.trace_id).to eq(publish_span.trace_id) + end + end + end +end diff --git a/spec/ddtrace/contrib/redis/quantize_spec.rb b/spec/ddtrace/contrib/redis/quantize_spec.rb new file mode 100644 index 00000000000..35cc2ef74c4 --- /dev/null +++ b/spec/ddtrace/contrib/redis/quantize_spec.rb @@ -0,0 +1,72 @@ +require 'spec_helper' + +require 'redis' +require 'hiredis' +require 'ddtrace/contrib/redis/quantize' + +RSpec.describe Datadog::Contrib::Redis::Quantize do + describe '#format_arg' do + subject(:output) { described_class.format_arg(arg) } + + context 'given' do + context 'nil' do + let(:arg) { nil } + it { is_expected.to eq('') } + end + + context 'an empty string' do + let(:arg) { '' } + it { is_expected.to eq('') } + end + + context 'a string under the limit' do + let(:arg) { 'HGETALL' } + it { is_expected.to eq(arg) } + end + + context 'a string up to limit' do + let(:arg) { 'A' * 50 } + it { is_expected.to eq(arg) } + end + + context 'a string over the limit by one' do + let(:arg) { 'B' * 101 } + it { is_expected.to eq('B' * 47 + '...') } + end + + context 'a string over the limit by a lot' do + let(:arg) { 'C' * 1000 } + it { is_expected.to eq('C' * 47 + '...') } + end + + context 'an object that can\'t be converted to a string' do + let(:arg) { object_class.new } + let(:object_class) { Class.new { def to_s; raise "can't make a string of me" end } } + it { is_expected.to eq('?') } + end + + context 'an invalid byte sequence' do + # \255 is off-limits https://en.wikipedia.org/wiki/UTF-8#Codepage_layout + let(:arg) { "SET foo bar\255" } + it { is_expected.to eq('SET foo bar') } + end + end + end + + describe '#format_command_args' do + subject(:output) { described_class.format_command_args(args) } + + context 'given an array' do + context 'of some basic values' do + let(:args) { [:set, 'KEY', 'VALUE'] } + it { is_expected.to eq('SET KEY VALUE') } + end + + context 'of many very long args (over the limit)' do + let(:args) { Array.new(20) { 'X' * 90 } } + it { expect(output.length).to eq(500) } + it { expect(output[496..499]).to eq('X...') } + end + end + end +end diff --git a/spec/ddtrace/contrib/redis/redis_spec.rb b/spec/ddtrace/contrib/redis/redis_spec.rb new file mode 100644 index 00000000000..c5cb605a9d2 --- /dev/null +++ b/spec/ddtrace/contrib/redis/redis_spec.rb @@ -0,0 +1,208 @@ +require 'spec_helper' + +require 'time' +require 'redis' +require 'hiredis' +require 'ddtrace' + +RSpec.describe 'Redis test' do + let(:tracer) { ::Datadog::Tracer.new(writer: FauxWriter.new) } + + def all_spans + tracer.writer.spans(:keep) + end + + before(:each) do + Datadog.configure do |c| + c.use :redis, tracer: tracer + end + end + + shared_examples_for 'a Redis driver' do |driver| + let(:redis) { Redis.new(host: host, port: port, driver: driver) } + let(:host) { '127.0.0.1' } + let(:port) { 46379 } + + let(:client) do + if Gem::Version.new(::Redis::VERSION) >= Gem::Version.new('4.0.0') + redis._client + else + redis.client + end + end + + let(:pin) { Datadog::Pin.get_from(client) } + + it { expect(pin).to_not be nil } + it { expect(pin.app_type).to eq('db') } + + shared_examples_for 'a span with common tags' do + it do + expect(span).to_not be nil + expect(span.get_tag('out.host')).to eq('127.0.0.1') + expect(span.get_tag('out.port')).to eq('46379') + expect(span.get_tag('out.redis_db')).to eq('0') + end + end + + context 'roundtrip' do + # Run a roundtrip + before(:each) do + expect(redis.set('FOO', 'bar')).to eq('OK') + expect(redis.get('FOO')).to eq('bar') + end + + it { expect(all_spans).to have(2).items } + + describe 'set span' do + subject(:span) { all_spans[-1] } + + it do + expect(span.name).to eq('redis.command') + expect(span.service).to eq('redis') + expect(span.resource).to eq('SET FOO bar') + expect(span.get_tag('redis.raw_command')).to eq('SET FOO bar') + end + + it_behaves_like 'a span with common tags' + end + + describe 'get span' do + subject(:span) { all_spans[0] } + + it do + expect(span.name).to eq('redis.command') + expect(span.service).to eq('redis') + expect(span.resource).to eq('GET FOO') + expect(span.get_tag('redis.raw_command')).to eq('GET FOO') + end + + it_behaves_like 'a span with common tags' + end + end + + context 'pipeline' do + before(:each) do + redis.pipelined do + responses << redis.set('v1', '0') + responses << redis.set('v2', '0') + responses << redis.incr('v1') + responses << redis.incr('v2') + responses << redis.incr('v2') + end + end + + let(:responses) { [] } + + it do + expect(responses.map(&:value)).to eq(['OK', 'OK', 1, 1, 2]) + expect(all_spans).to have(1).items + end + + describe 'span' do + subject(:span) { all_spans[-1] } + + it do + expect(span.get_metric('redis.pipeline_length')).to eq(5) + expect(span.name).to eq('redis.command') + expect(span.service).to eq('redis') + expect(span.resource).to eq("SET v1 0\nSET v2 0\nINCR v1\nINCR v2\nINCR v2") + expect(span.get_tag('redis.raw_command')).to eq("SET v1 0\nSET v2 0\nINCR v1\nINCR v2\nINCR v2") + end + + it_behaves_like 'a span with common tags' + end + end + + context 'error' do + subject(:bad_call) do + redis.call 'THIS_IS_NOT_A_REDIS_FUNC', 'THIS_IS_NOT_A_VALID_ARG' + end + + before(:each) do + expect { bad_call }.to raise_error(Redis::CommandError, "ERR unknown command 'THIS_IS_NOT_A_REDIS_FUNC'") + end + + it do + expect(all_spans).to have(1).items + end + + describe 'span' do + subject(:span) { all_spans[-1] } + + it do + expect(span.name).to eq('redis.command') + expect(span.service).to eq('redis') + expect(span.resource).to eq('THIS_IS_NOT_A_REDIS_FUNC THIS_IS_NOT_A_VALID_ARG') + expect(span.get_tag('redis.raw_command')).to eq('THIS_IS_NOT_A_REDIS_FUNC THIS_IS_NOT_A_VALID_ARG') + expect(span.status).to eq(1) + expect(span.get_tag('error.msg')).to eq("ERR unknown command 'THIS_IS_NOT_A_REDIS_FUNC'") + expect(span.get_tag('error.type')).to eq('Redis::CommandError') + expect(span.get_tag('error.stack').length).to be >= 3 + end + + it_behaves_like 'a span with common tags' + end + end + + context 'quantize' do + before(:each) do + expect(redis.set('K', 'x' * 500)).to eq('OK') + expect(redis.get('K')).to eq('x' * 500) + end + + it { expect(all_spans).to have(2).items } + + describe 'set span' do + subject(:span) { all_spans[-1] } + + it do + expect(span.name).to eq('redis.command') + expect(span.service).to eq('redis') + expect(span.resource).to eq('SET K ' + 'x' * 47 + '...') + expect(span.get_tag('redis.raw_command')).to eq('SET K ' + 'x' * 47 + '...') + end + + it_behaves_like 'a span with common tags' + end + + describe 'get span' do + subject(:span) { all_spans[-2] } + + it do + expect(span.name).to eq('redis.command') + expect(span.service).to eq('redis') + expect(span.resource).to eq('GET K') + expect(span.get_tag('redis.raw_command')).to eq('GET K') + end + + it_behaves_like 'a span with common tags' + end + end + + context 'service name' do + let(:services) { tracer.writer.services } + let(:service_name) { 'redis-test' } + + before(:each) do + redis.set 'FOO', 'bar' + tracer.writer.services # empty queue + Datadog.configure( + client, + service_name: service_name, + tracer: tracer, + app_type: Datadog::Ext::AppTypes::CACHE + ) + redis.set 'FOO', 'bar' + end + + it do + expect(services).to have(1).items + expect(services[service_name]).to eq({ 'app' => 'redis', 'app_type' => 'cache' }) + end + end + end + + it_behaves_like 'a Redis driver', :ruby + it_behaves_like 'a Redis driver', :hiredis +end From 64078dc29060026959e000d98c89eac2b821ee21 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 12 Feb 2018 15:29:20 -0500 Subject: [PATCH 14/72] Removed: Redis minitests in favor of RSpec. --- Rakefile | 25 ++- test/contrib/redis/integration_test.rb | 36 ----- test/contrib/redis/method_replaced_test.rb | 47 ------ test/contrib/redis/miniapp_test.rb | 95 ------------ test/contrib/redis/quantize_test.rb | 42 ------ test/contrib/redis/redis_test.rb | 167 --------------------- test/contrib/redis/test_helper.rb | 3 - 7 files changed, 12 insertions(+), 403 deletions(-) delete mode 100644 test/contrib/redis/integration_test.rb delete mode 100644 test/contrib/redis/method_replaced_test.rb delete mode 100644 test/contrib/redis/miniapp_test.rb delete mode 100644 test/contrib/redis/quantize_test.rb delete mode 100644 test/contrib/redis/redis_test.rb delete mode 100644 test/contrib/redis/test_helper.rb diff --git a/Rakefile b/Rakefile index 024eb80e85c..e6ed8de0a52 100644 --- a/Rakefile +++ b/Rakefile @@ -40,22 +40,22 @@ namespace :spec do end [ + :active_record, + :aws, + :dalli, :elasticsearch, - :http, - :redis, - :sinatra, - :sidekiq, - :rack, :faraday, :grape, :graphql, - :aws, - :sucker_punch, + :http, :mongodb, :racecar, + :rack, + :redis, :resque, - :active_record, - :dalli + :sidekiq, + :sinatra, + :sucker_punch ].each do |contrib| RSpec::Core::RakeTask.new(contrib) do |t| t.pattern = "spec/ddtrace/contrib/#{contrib}/*_spec.rb" @@ -66,7 +66,7 @@ end namespace :test do task all: [:main, :rails, :railsredis, :railssidekiq, :railsactivejob, - :elasticsearch, :http, :redis, :sidekiq, :sinatra, :monkey] + :elasticsearch, :http, :sidekiq, :sinatra, :monkey] Rake::TestTask.new(:main) do |t| t.libs << %w[test lib] @@ -116,7 +116,6 @@ namespace :test do :mongodb, :resque, :rack, - :redis, :resque, :sidekiq, :sinatra, @@ -208,7 +207,6 @@ task :ci do when 1 sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:elasticsearch' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:http' - sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:redis' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:sinatra' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:rack' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:grape' @@ -219,7 +217,6 @@ task :ci do sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:monkey' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:elasticsearch' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:http' - sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:redis' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:sinatra' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:rack' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:aws' @@ -232,9 +229,11 @@ task :ci do sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:faraday' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:graphql' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:racecar' + sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:redis' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:active_record' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:dalli' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:faraday' + sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:redis' when 2 sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake test:sidekiq' sh 'rvm $SIDEKIQ_OLD_VERSIONS --verbose do appraisal contrib-old rake test:sidekiq' diff --git a/test/contrib/redis/integration_test.rb b/test/contrib/redis/integration_test.rb deleted file mode 100644 index b71c87e300a..00000000000 --- a/test/contrib/redis/integration_test.rb +++ /dev/null @@ -1,36 +0,0 @@ -require 'time' -require 'contrib/redis/test_helper' -require 'helper' - -class RedisIntegrationTest < Minitest::Test - REDIS_HOST = '127.0.0.1'.freeze - REDIS_PORT = 46379 - - def setup - skip unless ENV['TEST_DATADOG_INTEGRATION'] # requires a running agent - - # Here we use the default tracer (to make a real integration test) - @tracer = Datadog::Tracer.new - Datadog.instance_variable_set(:@tracer, @tracer) - Datadog.configure do |c| - c.use :redis, tracer: @tracer - end - - @redis = Redis.new(host: REDIS_HOST, port: REDIS_PORT) - end - - def teardown - Datadog.configure do |c| - c.use :redis - end - end - - def test_setget - set_response = @redis.set 'FOO', 'bar' - assert_equal 'OK', set_response - get_response = @redis.get 'FOO' - assert_equal 'bar', get_response - try_wait_until(attempts: 30) { @tracer.writer.stats[:traces_flushed] >= 2 } - assert_operator(2, :<=, @tracer.writer.stats[:traces_flushed]) - end -end diff --git a/test/contrib/redis/method_replaced_test.rb b/test/contrib/redis/method_replaced_test.rb deleted file mode 100644 index d47c41cfed6..00000000000 --- a/test/contrib/redis/method_replaced_test.rb +++ /dev/null @@ -1,47 +0,0 @@ -require 'contrib/redis/test_helper' -require 'helper' - -class RedisMethodReplacedTest < Minitest::Test - # We want to make sure that the patcher works even when the patched methods - # have already been replaced by another library. - - REDIS_HOST = '127.0.0.1'.freeze - REDIS_PORT = 46379 - - def setup - Datadog.configure do |c| - c.use :redis - end - - ::Redis::Client.class_eval do - alias_method :call_original, :call - remove_method :call - def call(*args, &block) - @datadog_test_called ||= false - if @datadog_test_called - raise Minitest::Assertion, 'patched methods called in infinite loop' - end - @datadog_test_called = true - - call_original(*args, &block) - end - end - end - - def test_main - skip unless ENV['TEST_DATADOG_INTEGRATION'] # requires a running agent - - refute_nil(Redis::Client.instance_methods.find { |m| m == :call_without_datadog }) - - redis = Redis.new(host: REDIS_HOST, port: REDIS_PORT) - redis.call(['ping', 'hello world']) - end - - def teardown - ::Redis::Client.class_eval do - remove_method :call - alias_method :call, :call_original - remove_method :call_original - end - end -end diff --git a/test/contrib/redis/miniapp_test.rb b/test/contrib/redis/miniapp_test.rb deleted file mode 100644 index a542c3f1c5f..00000000000 --- a/test/contrib/redis/miniapp_test.rb +++ /dev/null @@ -1,95 +0,0 @@ -require 'time' -require 'contrib/redis/test_helper' -require 'helper' - -# RedisMiniAppTest tests and shows what you would typically do -# in a custom application, which is already traced. It shows -# how to have Redis spans be children of application spans. -class RedisMiniAppTest < Minitest::Test - REDIS_HOST = '127.0.0.1'.freeze - REDIS_PORT = 46379 - - def setup - Datadog.configure do |c| - c.use :redis - end - - @redis = Redis.new(host: REDIS_HOST, port: REDIS_PORT) - @tracer = get_test_tracer - Datadog.configure(client_from_driver(redis), tracer: tracer) - end - - def check_span_publish(span) - assert_equal('publish', span.name) - assert_equal('webapp', span.service) - assert_equal('/index', span.resource) - refute_equal(span.trace_id, span.span_id) - assert_equal(0, span.parent_id) - end - - def check_span_process(span, parent_id, trace_id) - assert_equal('process', span.name) - assert_equal('datalayer', span.service) - assert_equal('home', span.resource) - assert_equal(parent_id, span.parent_id) - assert_equal(trace_id, span.trace_id) - end - - def check_span_command(span, parent_id, trace_id) - assert_equal('redis.command', span.name) - assert_equal('redis', span.service) - assert_equal(parent_id, span.parent_id) - assert_equal(trace_id, span.trace_id) - end - - def test_miniapp - # now this is how you make sure that the redis spans are sub-spans - # of the apps parent spans: - tracer.trace('publish') do |span| - span.service = 'webapp' - span.resource = '/index' - tracer.trace('process') do |subspan| - subspan.service = 'datalayer' - subspan.resource = 'home' - redis.get 'data1' - redis.pipelined do - redis.set 'data2', 'something' - redis.get 'data2' - end - end - end - - spans = tracer.writer.spans - - # here we should get 4 spans, with : - # spans[3] being the parent of span[2] - # spand[2] being the parant of span[0] and span[1] - assert_equal(4, spans.length) - process, publish, redis_cmd1, redis_cmd2 = spans - # span[3] (publish) - # \ - # ------> span[2] (process) - # \ - # |-----> span[1] - # \-----> span[0] - check_span_publish publish - parent_id = publish.span_id - trace_id = publish.trace_id - check_span_process process, parent_id, trace_id - parent_id = process.span_id - check_span_command redis_cmd1, parent_id, trace_id - check_span_command redis_cmd2, parent_id, trace_id - end - - private - - attr_reader :tracer, :redis - - def client_from_driver(driver) - if Gem::Version.new(::Redis::VERSION) >= Gem::Version.new('4.0.0') - driver._client - else - driver.client - end - end -end diff --git a/test/contrib/redis/quantize_test.rb b/test/contrib/redis/quantize_test.rb deleted file mode 100644 index ac54934adf7..00000000000 --- a/test/contrib/redis/quantize_test.rb +++ /dev/null @@ -1,42 +0,0 @@ -require 'ddtrace/contrib/redis/quantize' -require 'contrib/redis/test_helper' -require 'helper' - -class Unstringable - def to_s - raise "can't make a string of me" - end -end - -class RedisQuantizeTest < Minitest::Test - def test_format_arg - expected = { '' => '', - 'HGETALL' => 'HGETALL', - 'A' * 50 => 'A' * 50, - 'B' * 101 => 'B' * 47 + '...', - 'C' * 1000 => 'C' * 47 + '...', - nil => '', - Unstringable.new => '?' } - expected.each do |k, v| - assert_equal(v, Datadog::Contrib::Redis::Quantize.format_arg(k)) - end - end - - def test_format_command_args - assert_equal('SET KEY VALUE', Datadog::Contrib::Redis::Quantize.format_command_args([:set, 'KEY', 'VALUE'])) - command_args = [] - 20.times { command_args << ('X' * 90) } - trimmed = Datadog::Contrib::Redis::Quantize.format_command_args(command_args) - assert_equal(500, trimmed.length) - assert_equal('X...', trimmed[496..499]) - end - - def test_invalid_byte_sequence_regression - # \255 is off-limits https://en.wikipedia.org/wiki/UTF-8#Codepage_layout - quantized = Datadog::Contrib::Redis::Quantize.format_arg( - "SET foo bar\255" - ) - - assert_equal('SET foo bar', quantized) - end -end diff --git a/test/contrib/redis/redis_test.rb b/test/contrib/redis/redis_test.rb deleted file mode 100644 index e74084c772b..00000000000 --- a/test/contrib/redis/redis_test.rb +++ /dev/null @@ -1,167 +0,0 @@ -require 'time' -require 'contrib/redis/test_helper' -require 'helper' - -# rubocop:disable Metrics/ClassLength -class RedisTest < Minitest::Test - REDIS_HOST = '127.0.0.1'.freeze - REDIS_PORT = 46379 - - def setup - @tracer = get_test_tracer - - Datadog.configure do |c| - c.use :redis, tracer: tracer - end - - @drivers = { - ruby: Redis.new(host: REDIS_HOST, port: REDIS_PORT, driver: :ruby), - hiredis: Redis.new(host: REDIS_HOST, port: REDIS_PORT, driver: :hiredis) - } - end - - def teardown - # Reset tracer to default (so we don't break other tests) - Datadog.configure do |c| - c.use :redis, tracer: Datadog.tracer - end - end - - def check_common_tags(span) - assert_equal('127.0.0.1', span.get_tag('out.host')) - assert_equal('46379', span.get_tag('out.port')) - assert_equal('0', span.get_tag('out.redis_db')) - end - - def roundtrip_set(driver, service) - set_response = driver.set 'FOO', 'bar' - assert_equal 'OK', set_response - spans = tracer.writer.spans - assert_operator(1, :<=, spans.length) - span = spans[-1] - check_common_tags(span) - assert_equal('redis.command', span.name) - assert_equal(service, span.service) - assert_equal('SET FOO bar', span.resource) - assert_equal('SET FOO bar', span.get_tag('redis.raw_command')) - end - - def roundtrip_get(driver, service) - get_response = driver.get 'FOO' - assert_equal 'bar', get_response - spans = tracer.writer.spans - assert_equal(1, spans.length) - span = spans[0] - check_common_tags(span) - assert_equal('redis.command', span.name) - assert_equal(service, span.service) - assert_equal('GET FOO', span.resource) - assert_equal('GET FOO', span.get_tag('redis.raw_command')) - end - - def test_roundtrip - drivers.each do |_d, driver| - pin = Datadog::Pin.get_from(client_from_driver(driver)) - refute_nil(pin) - assert_equal('db', pin.app_type) - roundtrip_set driver, 'redis' - roundtrip_get driver, 'redis' - end - end - - def test_pipeline - drivers.each do |d, driver| - responses = [] - driver.pipelined do - responses << driver.set('v1', '0') - responses << driver.set('v2', '0') - responses << driver.incr('v1') - responses << driver.incr('v2') - responses << driver.incr('v2') - end - assert_equal(['OK', 'OK', 1, 1, 2], responses.map(&:value)) - spans = tracer.writer.spans - assert_operator(1, :<=, spans.length) - check_connect_span(d, spans[0]) if spans.length >= 2 - span = spans[-1] - check_common_tags(span) - assert_equal(5, span.get_metric('redis.pipeline_length')) - assert_equal('redis.command', span.name) - assert_equal('redis', span.service) - assert_equal("SET v1 0\nSET v2 0\nINCR v1\nINCR v2\nINCR v2", span.resource) - assert_equal("SET v1 0\nSET v2 0\nINCR v1\nINCR v2\nINCR v2", span.get_tag('redis.raw_command')) - end - end - - def test_error - drivers.each do |_d, driver| - begin - driver.call 'THIS_IS_NOT_A_REDIS_FUNC', 'THIS_IS_NOT_A_VALID_ARG' - rescue StandardError => e - assert_kind_of(Redis::CommandError, e) - assert_equal("ERR unknown command 'THIS_IS_NOT_A_REDIS_FUNC'", e.to_s) - end - spans = tracer.writer.spans - assert_operator(1, :<=, spans.length) - span = spans[-1] - check_common_tags(span) - assert_equal('redis.command', span.name) - assert_equal('redis', span.service) - assert_equal('THIS_IS_NOT_A_REDIS_FUNC THIS_IS_NOT_A_VALID_ARG', span.resource) - assert_equal('THIS_IS_NOT_A_REDIS_FUNC THIS_IS_NOT_A_VALID_ARG', span.get_tag('redis.raw_command')) - assert_equal(1, span.status, 'this span should be flagged as an error') - assert_equal("ERR unknown command 'THIS_IS_NOT_A_REDIS_FUNC'", span.get_tag('error.msg')) - assert_equal('Redis::CommandError', span.get_tag('error.type')) - assert_operator(3, :<=, span.get_tag('error.stack').length) - end - end - - def test_quantize - drivers.each do |_d, driver| - driver.set 'K', 'x' * 500 - response = driver.get 'K' - assert_equal('x' * 500, response) - spans = tracer.writer.spans - assert_operator(2, :<=, spans.length) - get, set = spans[-2..-1] - check_common_tags(set) - assert_equal('redis.command', set.name) - assert_equal('redis', set.service) - assert_equal('SET K ' + 'x' * 47 + '...', set.resource) - assert_equal('SET K ' + 'x' * 47 + '...', set.get_tag('redis.raw_command')) - check_common_tags(get) - assert_equal('redis.command', get.name) - assert_equal('redis', get.service) - assert_equal('GET K', get.resource) - assert_equal('GET K', get.get_tag('redis.raw_command')) - end - end - - def test_service_name - driver = Redis.new(host: REDIS_HOST, port: REDIS_PORT, driver: :ruby) - driver.set 'FOO', 'bar' - tracer.writer.services # empty queue - Datadog.configure( - client_from_driver(driver), - service_name: 'redis-test', - tracer: tracer, - app_type: Datadog::Ext::AppTypes::CACHE - ) - driver.set 'FOO', 'bar' - services = tracer.writer.services - assert_equal(1, services.length) - assert_equal({ 'app' => 'redis', 'app_type' => 'cache' }, services['redis-test']) - end - - private - - attr_reader :tracer, :drivers - - def client_from_driver(driver) - if Gem::Version.new(::Redis::VERSION) >= Gem::Version.new('4.0.0') - driver._client - else - driver.client - end - end -end diff --git a/test/contrib/redis/test_helper.rb b/test/contrib/redis/test_helper.rb deleted file mode 100644 index 94be1c7a3ea..00000000000 --- a/test/contrib/redis/test_helper.rb +++ /dev/null @@ -1,3 +0,0 @@ -require 'ddtrace' -require 'redis' -require 'hiredis' From 2cd1d28194f9d8c95d4bb72b2e8da830f8e20249 Mon Sep 17 00:00:00 2001 From: Pierre Schambacher Date: Wed, 28 Feb 2018 10:25:14 -0800 Subject: [PATCH 15/72] Guard the rails core extensions against null Redis registry (#357) * Guard the rails core extensions against null Redis registry --- lib/ddtrace/contrib/rails/core_extensions.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ddtrace/contrib/rails/core_extensions.rb b/lib/ddtrace/contrib/rails/core_extensions.rb index 0f4a8776037..0cd2eb20f3b 100644 --- a/lib/ddtrace/contrib/rails/core_extensions.rb +++ b/lib/ddtrace/contrib/rails/core_extensions.rb @@ -310,7 +310,8 @@ def delete(*args, &block) end def self.reload_cache_store - return unless Datadog.registry[:redis].patched? + redis = Datadog.registry[:redis] + return unless redis && redis.patched? return unless defined?(::ActiveSupport::Cache::RedisStore) && defined?(::Rails.cache) && From 8d39d873ebcaadacef7e60bff60c57f15aefa9cc Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 28 Feb 2018 13:31:22 -0500 Subject: [PATCH 16/72] bumping version 0.12.0.beta1 => 0.12.0.beta2 --- lib/ddtrace/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ddtrace/version.rb b/lib/ddtrace/version.rb index b90db2c38ad..f4cd0fc4686 100644 --- a/lib/ddtrace/version.rb +++ b/lib/ddtrace/version.rb @@ -3,7 +3,7 @@ module VERSION MAJOR = 0 MINOR = 12 PATCH = 0 - PRE = 'beta1'.freeze + PRE = 'beta2'.freeze STRING = [MAJOR, MINOR, PATCH, PRE].compact.join('.') end From aeff57d1a2f362b0537e4c2bae892c4e1a226f0b Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 23 Feb 2018 14:53:51 -0500 Subject: [PATCH 17/72] Added: Spec for Rails middleware added after Datadog patching. --- spec/ddtrace/contrib/rails/middleware_spec.rb | 54 +++++++++++++++++-- .../contrib/rails/support/application.rb | 8 +-- spec/ddtrace/contrib/rails/support/base.rb | 2 +- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/spec/ddtrace/contrib/rails/middleware_spec.rb b/spec/ddtrace/contrib/rails/middleware_spec.rb index 7b8c565a802..4e60d742437 100644 --- a/spec/ddtrace/contrib/rails/middleware_spec.rb +++ b/spec/ddtrace/contrib/rails/middleware_spec.rb @@ -21,19 +21,61 @@ def all_spans tracer.writer.spans(:keep) end + RSpec::Matchers.define :have_kind_of_middleware do |expected| + match do |actual| + while actual + return true if actual.class <= expected + without_warnings { actual = actual.instance_variable_get(:@app) } + end + false + end + end + before(:each) do Datadog.configure do |c| - c.use :rack, tracer: tracer - c.use :rails, tracer: tracer + c.use :rack, rack_options + c.use :rails, rails_options end end + let(:rack_options) { { tracer: tracer } } + let(:rails_options) { { tracer: tracer } } + context 'with middleware' do before(:each) { get '/' } - let(:rails_middleware) { [middleware] } + context 'that does nothing' do + let(:middleware) do + stub_const('PassthroughMiddleware', Class.new do + def initialize(app) + @app = app + end + + def call(env) + @app.call(env) + end + end) + end + + context 'and added after tracing is enabled' do + before(:each) do + passthrough_middleware = middleware + rails_test_application.configure { config.app_middleware.use passthrough_middleware } + end + + context 'with #middleware_names' do + let(:rack_options) { super().merge!(middleware_names: true) } + + it do + expect(app).to have_kind_of_middleware(middleware) + expect(last_response).to be_ok + end + end + end + end context 'that raises an exception' do + let(:rails_middleware) { [middleware] } let(:middleware) do stub_const('RaiseExceptionMiddleware', Class.new do def initialize(app) @@ -48,6 +90,7 @@ def call(env) end it do + expect(app).to have_kind_of_middleware(middleware) expect(last_response).to be_server_error expect(all_spans.length).to be >= 2 end @@ -71,6 +114,7 @@ def call(env) end context 'that raises a known NotFound exception' do + let(:rails_middleware) { [middleware] } let(:middleware) do stub_const('RaiseNotFoundMiddleware', Class.new do def initialize(app) @@ -85,6 +129,7 @@ def call(env) end it do + expect(app).to have_kind_of_middleware(middleware) expect(last_response).to be_not_found expect(all_spans.length).to be >= 2 end @@ -117,6 +162,7 @@ def call(env) end context 'that raises a custom exception' do + let(:rails_middleware) { [middleware] } let(:error_class) do stub_const('CustomError', Class.new(StandardError) do def message @@ -142,6 +188,7 @@ def call(env) end it do + expect(app).to have_kind_of_middleware(middleware) expect(last_response).to be_server_error expect(all_spans.length).to be >= 2 end @@ -191,6 +238,7 @@ def call(env) end it do + expect(app).to have_kind_of_middleware(middleware) expect(last_response).to be_not_found expect(all_spans.length).to be >= 2 end diff --git a/spec/ddtrace/contrib/rails/support/application.rb b/spec/ddtrace/contrib/rails/support/application.rb index b6fac5197c4..e7b1fa3fe89 100644 --- a/spec/ddtrace/contrib/rails/support/application.rb +++ b/spec/ddtrace/contrib/rails/support/application.rb @@ -4,11 +4,7 @@ include_context 'Rails base application' let(:app) do - if Rails.version >= '3.2' - rails_test_application.to_app - else - rails_test_application - end + rails_test_application.instance end before(:each) do @@ -37,7 +33,7 @@ end elsif Rails.version >= '3.0' let(:rails_test_application) do - rails_base_application + stub_const('Rails3::Application', rails_base_application) end else logger.error 'A Rails app for this version is not found!' diff --git a/spec/ddtrace/contrib/rails/support/base.rb b/spec/ddtrace/contrib/rails/support/base.rb index 7e600e8f4af..a1fa2fb92ce 100644 --- a/spec/ddtrace/contrib/rails/support/base.rb +++ b/spec/ddtrace/contrib/rails/support/base.rb @@ -25,7 +25,7 @@ debug_mw = debug_middleware Proc.new do - config.middleware.insert_before 0, debug_mw + config.middleware.insert_after ActionDispatch::ShowExceptions, debug_mw middleware.each { |m| config.middleware.use m } end end From 5d0f5b069eb57856e7af4612a00e2a8cb63b260b Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 23 Feb 2018 15:48:25 -0500 Subject: [PATCH 18/72] Changed: Rails application to lazy initialize in specs. --- spec/ddtrace/contrib/rails/support/application.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/spec/ddtrace/contrib/rails/support/application.rb b/spec/ddtrace/contrib/rails/support/application.rb index e7b1fa3fe89..bfd5e3a8b3e 100644 --- a/spec/ddtrace/contrib/rails/support/application.rb +++ b/spec/ddtrace/contrib/rails/support/application.rb @@ -4,15 +4,21 @@ include_context 'Rails base application' let(:app) do + initialize_app! rails_test_application.instance end - before(:each) do + def initialize_app! # Reinitializing Rails applications generates a lot of warnings. without_warnings do # Initialize the application and stub Rails with the test app rails_test_application.test_initialize! end + + # Clear out any garbage spans generated by initialization + if defined?(tracer) + tracer.writer.spans if tracer.writer.class <= FauxWriter + end end if Rails.version < '4.0' From 1363341e140ee941869395f5c3f0f3c81db7afb6 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 23 Feb 2018 15:52:54 -0500 Subject: [PATCH 19/72] Added: middleware_names option for Rails. --- docs/GettingStarted.md | 1 + lib/ddtrace/contrib/rails/framework.rb | 1 + lib/ddtrace/contrib/rails/patcher.rb | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index aed8a6b6784..bd5055b5fac 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -81,6 +81,7 @@ Where `options` is an optional `Hash` that accepts the following parameters: | ``database_service`` | Database service name used when tracing database activity | ``-`` | | ``exception_controller`` | Class or Module which identifies a custom exception controller class. Tracer provides improved error behavior when it can identify custom exception controllers. By default, without this option, it 'guesses' what a custom exception controller looks like. Providing this option aids this identification. | ``nil`` | | ``distributed_tracing`` | Enables [distributed tracing](#Distributed_Tracing) so that this service trace is connected with a trace of another service if tracing headers are received | `false` | +| ``middleware_names`` | Enables any short-circuited middleware requests to display the middleware name as resource for the trace. | `false` | | ``template_base_path`` | Used when the template name is parsed. If you don't store your templates in the ``views/`` folder, you may need to change this value | ``views/`` | | ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | diff --git a/lib/ddtrace/contrib/rails/framework.rb b/lib/ddtrace/contrib/rails/framework.rb index 92bb65b8459..0122dce321f 100644 --- a/lib/ddtrace/contrib/rails/framework.rb +++ b/lib/ddtrace/contrib/rails/framework.rb @@ -29,6 +29,7 @@ def self.setup :rack, tracer: tracer, service_name: config[:service_name], + middleware_names: config[:middleware_names], distributed_tracing: config[:distributed_tracing] ) diff --git a/lib/ddtrace/contrib/rails/patcher.rb b/lib/ddtrace/contrib/rails/patcher.rb index 4f943b0f7d9..f698376f21d 100644 --- a/lib/ddtrace/contrib/rails/patcher.rb +++ b/lib/ddtrace/contrib/rails/patcher.rb @@ -10,6 +10,7 @@ module Patcher option :controller_service option :cache_service option :database_service + option :middleware_names, default: false option :distributed_tracing, default: false option :template_base_path, default: 'views/' option :exception_controller, default: nil From 2ffcf0270e45f551210f3f5be110b1398e9fdfeb Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 23 Feb 2018 15:53:24 -0500 Subject: [PATCH 20/72] Fixed: Spec for middleware added after Datadog configure. --- spec/ddtrace/contrib/rails/middleware_spec.rb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/spec/ddtrace/contrib/rails/middleware_spec.rb b/spec/ddtrace/contrib/rails/middleware_spec.rb index 4e60d742437..6eb813ca42b 100644 --- a/spec/ddtrace/contrib/rails/middleware_spec.rb +++ b/spec/ddtrace/contrib/rails/middleware_spec.rb @@ -33,17 +33,17 @@ def all_spans before(:each) do Datadog.configure do |c| - c.use :rack, rack_options - c.use :rails, rails_options + c.use :rack, rack_options if use_rack + c.use :rails, rails_options if use_rails end end + let(:use_rack) { true } let(:rack_options) { { tracer: tracer } } + let(:use_rails) { true } let(:rails_options) { { tracer: tracer } } context 'with middleware' do - before(:each) { get '/' } - context 'that does nothing' do let(:middleware) do stub_const('PassthroughMiddleware', Class.new do @@ -64,9 +64,11 @@ def call(env) end context 'with #middleware_names' do - let(:rack_options) { super().merge!(middleware_names: true) } + let(:use_rack) { false } + let(:rails_options) { super().merge!(middleware_names: true) } it do + get '/' expect(app).to have_kind_of_middleware(middleware) expect(last_response).to be_ok end @@ -75,6 +77,8 @@ def call(env) end context 'that raises an exception' do + before(:each) { get '/' } + let(:rails_middleware) { [middleware] } let(:middleware) do stub_const('RaiseExceptionMiddleware', Class.new do @@ -114,6 +118,8 @@ def call(env) end context 'that raises a known NotFound exception' do + before(:each) { get '/' } + let(:rails_middleware) { [middleware] } let(:middleware) do stub_const('RaiseNotFoundMiddleware', Class.new do @@ -162,6 +168,8 @@ def call(env) end context 'that raises a custom exception' do + before(:each) { get '/' } + let(:rails_middleware) { [middleware] } let(:error_class) do stub_const('CustomError', Class.new(StandardError) do From a33b8f68ffa689ea16b9d95446f42d71f274af25 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 23 Feb 2018 15:55:57 -0500 Subject: [PATCH 21/72] Fixed: Bad resource when RESPONSE_MIDDLEWARE missing (reverts to default behavior.) --- lib/ddtrace/contrib/rack/middlewares.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ddtrace/contrib/rack/middlewares.rb b/lib/ddtrace/contrib/rack/middlewares.rb index f90897fca7b..0818e3c1145 100644 --- a/lib/ddtrace/contrib/rack/middlewares.rb +++ b/lib/ddtrace/contrib/rack/middlewares.rb @@ -77,7 +77,7 @@ def call(env) end def resource_name_for(env, status) - if Datadog.configuration[:rack][:middleware_names] + if Datadog.configuration[:rack][:middleware_names] && env['RESPONSE_MIDDLEWARE'] "#{env['RESPONSE_MIDDLEWARE']}##{env['REQUEST_METHOD']}" else "#{env['REQUEST_METHOD']} #{status}".strip From 8b317d711bfe12e507b0a77e7ae99660cb81e937 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 23 Feb 2018 17:01:48 -0500 Subject: [PATCH 22/72] Changed: Defer Rack middleware patching to only when option is passed. --- lib/ddtrace/contrib/rack/patcher.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/ddtrace/contrib/rack/patcher.rb b/lib/ddtrace/contrib/rack/patcher.rb index 22160e6bca4..5cec3b844cc 100644 --- a/lib/ddtrace/contrib/rack/patcher.rb +++ b/lib/ddtrace/contrib/rack/patcher.rb @@ -17,12 +17,17 @@ module Patcher module_function def patch - return true if patched? + unless patched? + require_relative 'middlewares' + @patched = true + end - require_relative 'middlewares' - @patched = true + if !@middleware_patched && get_option(:middleware_names) + enable_middleware_names + @middleware_patched = true + end - enable_middleware_names if get_option(:middleware_names) + @patched || @middleware_patched end def patched? From 8e661680af331ec2ddb8612bfbc837b803115800 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 23 Feb 2018 17:02:47 -0500 Subject: [PATCH 23/72] Changed: Rack patching doesn't look for Rails application, it must be supplied as an option. --- docs/GettingStarted.md | 3 ++- lib/ddtrace/contrib/rack/patcher.rb | 10 ++-------- lib/ddtrace/contrib/rails/framework.rb | 1 + 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index bd5055b5fac..993218b2fd5 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -141,7 +141,8 @@ Where `options` is an optional `Hash` that accepts the following parameters: | --- | --- | --- | | ``service_name`` | Service name used when tracing application requests | rack | | ``distributed_tracing`` | Enables [distributed tracing](#Distributed_Tracing) so that this service trace is connected with a trace of another service if tracing headers are received | `false` | -| ``middleware_names`` | Enable this if you want to use the middleware classes as the resource names for `rack` spans | ``false`` | +| ``middleware_names`` | Enable this if you want to use the middleware classes as the resource names for `rack` spans. Must provide the ``application`` option with it. | ``false`` | +| ``application`` | Your Rack application. Necessary for enabling middleware resource names. | ``nil`` | | ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | ## Other libraries diff --git a/lib/ddtrace/contrib/rack/patcher.rb b/lib/ddtrace/contrib/rack/patcher.rb index 5cec3b844cc..11e9d5e9960 100644 --- a/lib/ddtrace/contrib/rack/patcher.rb +++ b/lib/ddtrace/contrib/rack/patcher.rb @@ -22,7 +22,7 @@ def patch @patched = true end - if !@middleware_patched && get_option(:middleware_names) + if !@middleware_patched && get_option(:middleware_names) && get_option(:application) enable_middleware_names @middleware_patched = true end @@ -35,8 +35,7 @@ def patched? end def enable_middleware_names - root = get_option(:application) || rails_app - retain_middleware_name(root) + retain_middleware_name(get_option(:application)) rescue => e # We can safely ignore these exceptions since they happen only in the # context of middleware patching outside a Rails server process (eg. a @@ -45,11 +44,6 @@ def enable_middleware_names Tracer.log.debug("Error patching middleware stack: #{e}") end - def rails_app - return unless Datadog.registry[:rails].compatible? - ::Rails.application.app - end - def retain_middleware_name(middleware) return unless middleware && middleware.respond_to?(:call) diff --git a/lib/ddtrace/contrib/rails/framework.rb b/lib/ddtrace/contrib/rails/framework.rb index 0122dce321f..413093923de 100644 --- a/lib/ddtrace/contrib/rails/framework.rb +++ b/lib/ddtrace/contrib/rails/framework.rb @@ -28,6 +28,7 @@ def self.setup Datadog.configuration.use( :rack, tracer: tracer, + application: ::Rails.application, service_name: config[:service_name], middleware_names: config[:middleware_names], distributed_tracing: config[:distributed_tracing] From c139c3ca0f99859b2b20194738f14694a01439ac Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 26 Feb 2018 10:43:49 -0500 Subject: [PATCH 24/72] Fixed: Rack test for middleware names. --- test/contrib/rack/resource_name_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/contrib/rack/resource_name_test.rb b/test/contrib/rack/resource_name_test.rb index 945ce803073..9e5ded07da6 100644 --- a/test/contrib/rack/resource_name_test.rb +++ b/test/contrib/rack/resource_name_test.rb @@ -14,6 +14,7 @@ def setup end.to_app remove_patch!(:rack) + Datadog.registry[:rack].instance_variable_set('@middleware_patched', false) Datadog.configuration.use( :rack, middleware_names: true, From b7e4d1826bb6055d4ce31519387207fb1252780d Mon Sep 17 00:00:00 2001 From: David Elner Date: Thu, 1 Mar 2018 14:16:53 -0500 Subject: [PATCH 25/72] Added: Deprecation warning when middleware_names enabled without application. --- lib/ddtrace/contrib/rack/patcher.rb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/ddtrace/contrib/rack/patcher.rb b/lib/ddtrace/contrib/rack/patcher.rb index 11e9d5e9960..cc466e0dc10 100644 --- a/lib/ddtrace/contrib/rack/patcher.rb +++ b/lib/ddtrace/contrib/rack/patcher.rb @@ -22,9 +22,16 @@ def patch @patched = true end - if !@middleware_patched && get_option(:middleware_names) && get_option(:application) - enable_middleware_names - @middleware_patched = true + if !@middleware_patched && get_option(:middleware_names) + if get_option(:application) + enable_middleware_names + @middleware_patched = true + else + Datadog::Tracer.log.warn(%( + Rack :middleware_names requires you to also pass :application. + Middleware names have NOT been patched; please provide :application. + e.g. use: :rack, middleware_names: true, application: my_rack_app).freeze) + end end @patched || @middleware_patched From 6ce31865d7b79b039acac78fe5d4f546fff35fc7 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 7 Mar 2018 13:12:43 -0500 Subject: [PATCH 26/72] Refactored: :datadog_rack_request_span to 'datadog_rack_request_span' in Rack. --- lib/ddtrace/contrib/grape/endpoint.rb | 2 +- lib/ddtrace/contrib/rack/middlewares.rb | 8 +++++++- test/contrib/rack/helpers.rb | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/ddtrace/contrib/grape/endpoint.rb b/lib/ddtrace/contrib/grape/endpoint.rb index 3d0d173c9cd..25c7578d6c6 100644 --- a/lib/ddtrace/contrib/grape/endpoint.rb +++ b/lib/ddtrace/contrib/grape/endpoint.rb @@ -71,7 +71,7 @@ def self.endpoint_run(name, start, finish, id, payload) span.resource = resource # set the request span resource if it's a `rack.request` span - request_span = payload[:env][:datadog_rack_request_span] + request_span = payload[:env][Datadog::Contrib::Rack::TraceMiddleware::RACK_REQUEST_SPAN] if !request_span.nil? && request_span.name == 'rack.request' request_span.resource = resource end diff --git a/lib/ddtrace/contrib/rack/middlewares.rb b/lib/ddtrace/contrib/rack/middlewares.rb index 0818e3c1145..7363f70fa51 100644 --- a/lib/ddtrace/contrib/rack/middlewares.rb +++ b/lib/ddtrace/contrib/rack/middlewares.rb @@ -13,6 +13,8 @@ module Rack # application. If request tags are not set by the app, they will be set using # information available at the Rack level. class TraceMiddleware + RACK_REQUEST_SPAN = 'datadog.rack_request_span'.freeze + def initialize(app) @app = app end @@ -35,7 +37,11 @@ def call(env) # start a new request span and attach it to the current Rack environment; # we must ensure that the span `resource` is set later request_span = tracer.trace('rack.request', trace_options) - env[:datadog_rack_request_span] = request_span + env[RACK_REQUEST_SPAN] = request_span + + # TODO: For backwards compatibility; this attribute is deprecated. + # Will be removed in version 0.13.0. + env[:datadog_rack_request_span] = env[RACK_REQUEST_SPAN] # Copy the original env, before the rest of the stack executes. # Values may change; we want values before that happens. diff --git a/test/contrib/rack/helpers.rb b/test/contrib/rack/helpers.rb index 1a7f856777c..5d803005680 100644 --- a/test/contrib/rack/helpers.rb +++ b/test/contrib/rack/helpers.rb @@ -40,7 +40,7 @@ def app run(proc do |env| # this should be considered a web framework that can alter # the request span after routing / controller processing - request_span = env[:datadog_rack_request_span] + request_span = env[Datadog::Contrib::Rack::TraceMiddleware::RACK_REQUEST_SPAN] request_span.resource = 'GET /app/' request_span.set_tag('http.method', 'GET_V2') request_span.set_tag('http.status_code', 201) @@ -54,7 +54,7 @@ def app run(proc do |env| # this should be considered a web framework that can alter # the request span after routing / controller processing - request_span = env[:datadog_rack_request_span] + request_span = env[Datadog::Contrib::Rack::TraceMiddleware::RACK_REQUEST_SPAN] request_span.status = 1 request_span.set_tag('error.stack', 'Handled exception') @@ -66,7 +66,7 @@ def app run(proc do |env| # this should be considered a web framework that can alter # the request span after routing / controller processing - request_span = env[:datadog_rack_request_span] + request_span = env[Datadog::Contrib::Rack::TraceMiddleware::RACK_REQUEST_SPAN] request_span.set_tag('error.stack', 'Handled exception') [500, { 'Content-Type' => 'text/html' }, 'OK'] From 7edd4583842202bd30b647e0fa1e8787799548de Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 7 Mar 2018 13:13:08 -0500 Subject: [PATCH 27/72] Added: Deprecation warning for :datadog_rack_request_span. --- lib/ddtrace/contrib/rack/middlewares.rb | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/ddtrace/contrib/rack/middlewares.rb b/lib/ddtrace/contrib/rack/middlewares.rb index 7363f70fa51..824e9c2fab8 100644 --- a/lib/ddtrace/contrib/rack/middlewares.rb +++ b/lib/ddtrace/contrib/rack/middlewares.rb @@ -43,6 +43,9 @@ def call(env) # Will be removed in version 0.13.0. env[:datadog_rack_request_span] = env[RACK_REQUEST_SPAN] + # Add deprecation warnings + add_deprecation_warnings(env) + # Copy the original env, before the rest of the stack executes. # Values may change; we want values before that happens. original_env = env.dup @@ -145,6 +148,34 @@ def get_request_id(headers, env) headers ||= {} headers['X-Request-Id'] || headers['X-Request-ID'] || env['HTTP_X_REQUEST_ID'] end + + private + + REQUEST_SPAN_DEPRECATION_WARNING = %( + :datadog_rack_request_span is considered an internal symbol in the Rack env, + and has been been DEPRECATED. Public support for its usage is discontinued. + If you need the Rack request span, try using `Datadog.tracer.active_span`. + This key will be removed in version 0.13.0).freeze + + def add_deprecation_warnings(env) + env.instance_eval do + def [](key) + if key == :datadog_rack_request_span && !@request_span_warning_issued + Datadog::Tracer.log.warn(REQUEST_SPAN_DEPRECATION_WARNING) + @request_span_warning_issued = true + end + super + end + + def []=(key, value) + if key == :datadog_rack_request_span && !@request_span_warning_issued + Datadog::Tracer.log.warn(REQUEST_SPAN_DEPRECATION_WARNING) + @request_span_warning_issued = true + end + super + end + end + end end end end From 64d94d2d04d11d4625af9a001b8f60c66f21fd19 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 21 Mar 2018 17:13:32 -0400 Subject: [PATCH 28/72] Changed: Pin mysql2 < 0.5.0, to prevent CI build from breaking. --- Appraisals | 6 +++--- gemfiles/contrib.gemfile | 2 +- gemfiles/rails4_mysql2.gemfile | 2 +- gemfiles/rails5_mysql2.gemfile | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Appraisals b/Appraisals index 0f04bf285df..8ea400447ab 100644 --- a/Appraisals +++ b/Appraisals @@ -57,7 +57,7 @@ if RUBY_VERSION < '2.4.0' && RUBY_PLATFORM != 'java' if RUBY_VERSION >= '2.1.10' appraise 'rails4-mysql2' do gem 'rails', '4.2.7.1' - gem 'mysql2', platform: :ruby + gem 'mysql2', '< 0.5', platform: :ruby gem 'activerecord-jdbcmysql-adapter', platform: :jruby end @@ -79,7 +79,7 @@ if RUBY_VERSION < '2.4.0' && RUBY_PLATFORM != 'java' if RUBY_VERSION >= '2.2.2' appraise 'rails5-mysql2' do gem 'rails', '5.0.1' - gem 'mysql2', platform: :ruby + gem 'mysql2', '< 0.5', platform: :ruby end appraise 'rails5-postgres' do @@ -129,7 +129,7 @@ if RUBY_VERSION >= '2.2.2' && RUBY_PLATFORM != 'java' gem 'dalli' gem 'resque', '< 2.0' gem 'racecar', '>= 0.3.5' - gem 'mysql2', platform: :ruby + gem 'mysql2', '< 0.5', platform: :ruby end else appraise 'contrib-old' do diff --git a/gemfiles/contrib.gemfile b/gemfiles/contrib.gemfile index 9215a21d9d3..230818d6d58 100644 --- a/gemfiles/contrib.gemfile +++ b/gemfiles/contrib.gemfile @@ -19,6 +19,6 @@ gem "sucker_punch" gem "dalli" gem "resque", "< 2.0" gem "racecar", ">= 0.3.5" -gem "mysql2", platform: :ruby +gem "mysql2", "< 0.5", platform: :ruby gemspec path: "../" diff --git a/gemfiles/rails4_mysql2.gemfile b/gemfiles/rails4_mysql2.gemfile index e9fcaf8f08b..2f0de14c5f9 100644 --- a/gemfiles/rails4_mysql2.gemfile +++ b/gemfiles/rails4_mysql2.gemfile @@ -4,7 +4,7 @@ source "https://rubygems.org" gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master" gem "rails", "4.2.7.1" -gem "mysql2", platform: :ruby +gem "mysql2", "< 0.5", platform: :ruby gem "activerecord-jdbcmysql-adapter", platform: :jruby gemspec path: "../" diff --git a/gemfiles/rails5_mysql2.gemfile b/gemfiles/rails5_mysql2.gemfile index 40b27dd42da..cca2c942fa9 100644 --- a/gemfiles/rails5_mysql2.gemfile +++ b/gemfiles/rails5_mysql2.gemfile @@ -4,6 +4,6 @@ source "https://rubygems.org" gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master" gem "rails", "5.0.1" -gem "mysql2", platform: :ruby +gem "mysql2", "< 0.5", platform: :ruby gemspec path: "../" From bf7cb7764321eb7ed3212f6e25840fd59a6bd858 Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 20 Mar 2018 16:43:52 -0400 Subject: [PATCH 29/72] Added: ActiveSupport::Notifications::Subscriber module. --- Rakefile | 3 +- .../notifications/subscriber.rb | 63 ++++++++ .../notifications/subscription.rb | 66 ++++++++ .../notifications/subscriber_spec.rb | 131 ++++++++++++++++ .../notifications/subscription_spec.rb | 142 ++++++++++++++++++ 5 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 lib/ddtrace/contrib/active_support/notifications/subscriber.rb create mode 100644 lib/ddtrace/contrib/active_support/notifications/subscription.rb create mode 100644 spec/ddtrace/contrib/active_support/notifications/subscriber_spec.rb create mode 100644 spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb diff --git a/Rakefile b/Rakefile index 4e781755002..375eab8b0df 100644 --- a/Rakefile +++ b/Rakefile @@ -41,6 +41,7 @@ namespace :spec do [ :active_record, + :active_support, :aws, :dalli, :elasticsearch, @@ -58,7 +59,7 @@ namespace :spec do :sucker_punch ].each do |contrib| RSpec::Core::RakeTask.new(contrib) do |t| - t.pattern = "spec/ddtrace/contrib/#{contrib}/*_spec.rb" + t.pattern = "spec/ddtrace/contrib/#{contrib}/**/*_spec.rb" end end end diff --git a/lib/ddtrace/contrib/active_support/notifications/subscriber.rb b/lib/ddtrace/contrib/active_support/notifications/subscriber.rb new file mode 100644 index 00000000000..7779f67d4fd --- /dev/null +++ b/lib/ddtrace/contrib/active_support/notifications/subscriber.rb @@ -0,0 +1,63 @@ +require 'set' +require 'ddtrace/contrib/active_support/notifications/subscription' + +module Datadog + module Contrib + module ActiveSupport + module Notifications + module Subscriber + def self.included(base) + base.send(:extend, ClassMethods) + end + + module ClassMethods + # Returns a list of subscriptions created for this class. + def subscriptions + @subscriptions ||= Set.new + end + + # Returns whether subscriptions have been activated, via #subscribe! + def subscribed? + subscribed == true + end + + protected + + # Defines a callback for when subscribe! is called. + # Should contain subscription setup, defined by the inheriting class. + def on_subscribe(&block) + @on_subscribe_block = block + end + + # Runs the on_subscribe callback once, to activate subscriptions. + # Should be triggered by the inheriting class. + def subscribe! + return subscribed? if subscribed? || on_subscribe_block.nil? + on_subscribe_block.call + @subscribed = true + end + + # Creates a subscription and immediately activates it. + def subscribe(pattern, span_name, options = {}, tracer = Datadog.tracer, &block) + subscription(span_name, options, tracer, &block).tap do |subscription| + subscription.subscribe(pattern) + end + end + + # Creates a subscription without activating it. + # Subscription is added to the inheriting class' list of subscriptions. + def subscription(span_name, options = {}, tracer = Datadog.tracer, &block) + Subscription.new(tracer, span_name, options, &block).tap do |subscription| + subscriptions << subscription + end + end + + private + + attr_reader :subscribed, :on_subscribe_block + end + end + end + end + end +end diff --git a/lib/ddtrace/contrib/active_support/notifications/subscription.rb b/lib/ddtrace/contrib/active_support/notifications/subscription.rb new file mode 100644 index 00000000000..836be425ad8 --- /dev/null +++ b/lib/ddtrace/contrib/active_support/notifications/subscription.rb @@ -0,0 +1,66 @@ +require 'active_support/notifications' + +module Datadog + module Contrib + module ActiveSupport + module Notifications + class Subscription + def initialize(tracer, span_name, options, &block) + @tracer = tracer + @span_name = span_name + @options = options + @block = block + end + + def start(_name, _id, _payload) + ensure_clean_context! + @tracer.trace(@span_name, @options) + end + + def finish(name, id, payload) + span = @tracer.active_span + + # The subscriber block needs to remember to set the name of the span. + @block.call(span, name, id, payload) + + span.finish + end + + def subscribe(pattern) + return false if subscribers.key?(pattern) + subscribers[pattern] = ::ActiveSupport::Notifications.subscribe(pattern, self) + true + end + + def unsubscribe(pattern) + return false unless subscribers.key?(pattern) + ::ActiveSupport::Notifications.unsubscribe(subscribers[pattern]) + subscribers.delete(pattern) + true + end + + def unsubscribe_all + return false if subscribers.empty? + subscribers.keys.each { |pattern| unsubscribe(pattern) } + true + end + + protected + + # Pattern => ActiveSupport:Notifications::Subscribers + def subscribers + @subscribers ||= {} + end + + private + + def ensure_clean_context! + return unless @tracer.call_context.current_span + + @tracer.provider.context = Context.new + end + end + end + end + end +end diff --git a/spec/ddtrace/contrib/active_support/notifications/subscriber_spec.rb b/spec/ddtrace/contrib/active_support/notifications/subscriber_spec.rb new file mode 100644 index 00000000000..84e47559d3f --- /dev/null +++ b/spec/ddtrace/contrib/active_support/notifications/subscriber_spec.rb @@ -0,0 +1,131 @@ +require 'spec_helper' +require 'ddtrace' + +require 'ddtrace/contrib/active_support/notifications/subscriber' + +RSpec.describe Datadog::Contrib::ActiveSupport::Notifications::Subscriber do + describe 'implemented' do + subject(:test_class) do + Class.new.tap do |klass| + klass.send(:include, described_class) + end + end + + describe 'class' do + describe 'behavior' do + describe '#subscriptions' do + subject(:subscriptions) { test_class.subscriptions } + + context 'when no subscriptions have been created' do + it { is_expected.to be_empty } + end + + context 'when a subscription has been created' do + it do + subscription = test_class.send( + :subscription, + double('span name'), + double('options'), + double('tracer'), + &Proc.new { } + ) + + is_expected.to contain_exactly(subscription) + end + end + end + + describe '#subscribed?' do + subject(:subscribed) { test_class.subscribed? } + + context 'when #subscribe! hasn\'t been called' do + it { is_expected.to be false } + end + + context 'after #subscribe! has been called' do + before(:each) do + test_class.send(:on_subscribe, &Proc.new { }) + test_class.send(:subscribe!) + end + + it { is_expected.to be true } + end + end + + context 'that is protected' do + describe '#subscribe!' do + subject(:result) { test_class.send(:subscribe!) } + + context 'when #on_subscribe' do + context 'is defined' do + let(:on_subscribe_block) { Proc.new { spy.call } } + let(:spy) { double(:spy) } + + before(:each) { test_class.send(:on_subscribe, &on_subscribe_block) } + + it do + expect(spy).to receive(:call) + is_expected.to be true + end + + context 'but has already been called once' do + before(:each) do + allow(spy).to receive(:call) + test_class.send(:subscribe!) + end + + it do + expect(spy).to_not receive(:call) + is_expected.to be true + end + end + end + + context 'is not defined' do + it { is_expected.to be false } + end + end + end + + describe '#subscribe' do + subject(:subscription) { test_class.send(:subscribe, pattern, span_name, options, tracer, &block) } + let(:pattern) { double('pattern') } + let(:span_name) { double('span name') } + let(:options) { double('options') } + let(:tracer) { double('tracer') } + let(:block) { Proc.new { } } + + before(:each) do + expect(Datadog::Contrib::ActiveSupport::Notifications::Subscription).to receive(:new) + .with(tracer, span_name, options) + .and_call_original + + expect_any_instance_of(Datadog::Contrib::ActiveSupport::Notifications::Subscription).to receive(:subscribe) + .with(pattern) + end + + it { is_expected.to be_a_kind_of(Datadog::Contrib::ActiveSupport::Notifications::Subscription) } + it { expect(test_class.subscriptions).to contain_exactly(subscription) } + end + + describe '#subscription' do + subject(:subscription) { test_class.send(:subscription, span_name, options, tracer, &block) } + let(:span_name) { double('span name') } + let(:options) { double('options') } + let(:tracer) { double('tracer') } + let(:block) { Proc.new { } } + + before(:each) do + expect(Datadog::Contrib::ActiveSupport::Notifications::Subscription).to receive(:new) + .with(tracer, span_name, options) + .and_call_original + end + + it { is_expected.to be_a_kind_of(Datadog::Contrib::ActiveSupport::Notifications::Subscription) } + it { expect(test_class.subscriptions).to contain_exactly(subscription) } + end + end + end + end + end +end diff --git a/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb b/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb new file mode 100644 index 00000000000..23301f37391 --- /dev/null +++ b/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb @@ -0,0 +1,142 @@ +require 'spec_helper' +require 'ddtrace' + +require 'ddtrace/contrib/active_support/notifications/subscription' + +RSpec.describe Datadog::Contrib::ActiveSupport::Notifications::Subscription do + describe 'instance' do + subject(:subscription) { described_class.new(tracer, span_name, options, &block) } + let(:tracer) { instance_double(Datadog::Tracer) } + let(:span_name) { double('span_name') } + let(:options) { double('options') } + let(:block) do + Proc.new do |span, name, id, payload| + spy.call(span, name, id, payload) + end + end + let(:spy) { double('spy') } + + describe 'behavior' do + describe '#start' do + subject(:result) { subscription.start(name, id, payload) } + let(:name) { double('name') } + let(:id) { double('id') } + let(:payload) { double('payload') } + + let(:span) { instance_double(Datadog::Span) } + + it do + expect(tracer).to receive(:trace).with(span_name, options).and_return(span) + is_expected.to be(span) + end + end + + describe '#finish' do + subject(:result) { subscription.finish(name, id, payload) } + let(:name) { double('name') } + let(:id) { double('id') } + let(:payload) { double('payload') } + + let(:span) { instance_double(Datadog::Span) } + + it do + expect(tracer).to receive(:active_span).and_return(span).ordered + expect(spy).to receive(:call).with(span, name, id, payload).ordered + expect(span).to receive(:finish).and_return(span).ordered + is_expected.to be(span) + end + end + + describe '#subscribe' do + subject(:result) { subscription.subscribe(pattern) } + let(:pattern) { double('pattern') } + + let(:active_support_subscriber) { double('ActiveSupport subscriber') } + + context 'when not already subscribed to the pattern' do + it do + expect(ActiveSupport::Notifications).to receive(:subscribe) + .with(pattern, subscription) + .and_return(active_support_subscriber) + + is_expected.to be true + expect(subscription.send(:subscribers)).to include(pattern => active_support_subscriber) + end + end + + context 'when already subscribed to the pattern' do + before(:each) do + allow(ActiveSupport::Notifications).to receive(:subscribe) + .with(pattern, subscription) + .and_return(active_support_subscriber) + + subscription.subscribe(pattern) + end + + it { is_expected.to be false } + end + end + + describe '#unsubscribe' do + subject(:result) { subscription.unsubscribe(pattern) } + let(:pattern) { double('pattern') } + + let(:active_support_subscriber) { double('ActiveSupport subscriber') } + + context 'when not already subscribed to the pattern' do + it { is_expected.to be false } + end + + context 'when already subscribed to the pattern' do + before(:each) do + allow(ActiveSupport::Notifications).to receive(:subscribe) + .with(pattern, subscription) + .and_return(active_support_subscriber) + + subscription.subscribe(pattern) + end + + it do + expect(subscription.send(:subscribers)).to have(1).items + expect(ActiveSupport::Notifications).to receive(:unsubscribe) + .with(active_support_subscriber) + + is_expected.to be true + expect(subscription.send(:subscribers)).to be_empty + end + end + end + + describe '#unsubscribe_all' do + subject(:result) { subscription.unsubscribe_all } + + let(:active_support_subscriber) { double('ActiveSupport subscriber') } + + context 'when not already subscribed to the pattern' do + it { is_expected.to be false } + end + + context 'when already subscribed to the pattern' do + before(:each) do + allow(ActiveSupport::Notifications).to receive(:subscribe) + .with(kind_of(String), subscription) + .and_return(active_support_subscriber) + + subscription.subscribe('pattern 1') + subscription.subscribe('pattern 2') + end + + it do + expect(subscription.send(:subscribers)).to have(2).items + expect(ActiveSupport::Notifications).to receive(:unsubscribe) + .with(active_support_subscriber) + .twice + + is_expected.to be true + expect(subscription.send(:subscribers)).to be_empty + end + end + end + end + end +end From eb5750161ea3849918d5cdfdbe6cca0af7a4183c Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 21 Mar 2018 13:03:49 -0400 Subject: [PATCH 30/72] Changed: Subscription to expose some read-only properties. --- .../active_support/notifications/subscription.rb | 15 ++++++++++----- .../notifications/subscription_spec.rb | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/ddtrace/contrib/active_support/notifications/subscription.rb b/lib/ddtrace/contrib/active_support/notifications/subscription.rb index 836be425ad8..59a6b366b76 100644 --- a/lib/ddtrace/contrib/active_support/notifications/subscription.rb +++ b/lib/ddtrace/contrib/active_support/notifications/subscription.rb @@ -5,7 +5,13 @@ module Contrib module ActiveSupport module Notifications class Subscription + attr_reader \ + :tracer, + :span_name, + :options + def initialize(tracer, span_name, options, &block) + raise ArgumentError.new('Must be given a block!') unless block_given? @tracer = tracer @span_name = span_name @options = options @@ -14,11 +20,11 @@ def initialize(tracer, span_name, options, &block) def start(_name, _id, _payload) ensure_clean_context! - @tracer.trace(@span_name, @options) + tracer.trace(@span_name, @options) end def finish(name, id, payload) - span = @tracer.active_span + span = tracer.active_span # The subscriber block needs to remember to set the name of the span. @block.call(span, name, id, payload) @@ -55,9 +61,8 @@ def subscribers private def ensure_clean_context! - return unless @tracer.call_context.current_span - - @tracer.provider.context = Context.new + return unless tracer.call_context.current_span + tracer.provider.context = Context.new end end end diff --git a/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb b/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb index 23301f37391..470377d6810 100644 --- a/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb +++ b/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Datadog::Contrib::ActiveSupport::Notifications::Subscription do describe 'instance' do subject(:subscription) { described_class.new(tracer, span_name, options, &block) } - let(:tracer) { instance_double(Datadog::Tracer) } + let(:tracer) { ::Datadog::Tracer.new(writer: FauxWriter.new) } let(:span_name) { double('span_name') } let(:options) { double('options') } let(:block) do From 0ba215bd4e3043f58207ace0532a24202d1a9af1 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 21 Mar 2018 13:27:56 -0400 Subject: [PATCH 31/72] Added: Some descriptions to ActiveSupport::Notifications. --- lib/ddtrace/contrib/active_support/notifications/subscriber.rb | 3 +++ .../contrib/active_support/notifications/subscription.rb | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/ddtrace/contrib/active_support/notifications/subscriber.rb b/lib/ddtrace/contrib/active_support/notifications/subscriber.rb index 7779f67d4fd..4f0f5fc95d2 100644 --- a/lib/ddtrace/contrib/active_support/notifications/subscriber.rb +++ b/lib/ddtrace/contrib/active_support/notifications/subscriber.rb @@ -5,11 +5,14 @@ module Datadog module Contrib module ActiveSupport module Notifications + # For classes that listen to ActiveSupport::Notification events. + # Creates subscriptions that are wrapped with tracing. module Subscriber def self.included(base) base.send(:extend, ClassMethods) end + # Class methods that are implemented in the inheriting class. module ClassMethods # Returns a list of subscriptions created for this class. def subscriptions diff --git a/lib/ddtrace/contrib/active_support/notifications/subscription.rb b/lib/ddtrace/contrib/active_support/notifications/subscription.rb index 59a6b366b76..1280afbb095 100644 --- a/lib/ddtrace/contrib/active_support/notifications/subscription.rb +++ b/lib/ddtrace/contrib/active_support/notifications/subscription.rb @@ -4,6 +4,7 @@ module Datadog module Contrib module ActiveSupport module Notifications + # An ActiveSupport::Notification subscription that wraps events with tracing. class Subscription attr_reader \ :tracer, @@ -11,7 +12,7 @@ class Subscription :options def initialize(tracer, span_name, options, &block) - raise ArgumentError.new('Must be given a block!') unless block_given? + raise ArgumentError, 'Must be given a block!' unless block_given? @tracer = tracer @span_name = span_name @options = options From 30cc9e2a2d44229e41feafe308f8070214b2c4b0 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 21 Mar 2018 15:45:44 -0400 Subject: [PATCH 32/72] Added: nil check for ActiveSupport::Notification subscription finish event. --- .../active_support/notifications/subscription.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/ddtrace/contrib/active_support/notifications/subscription.rb b/lib/ddtrace/contrib/active_support/notifications/subscription.rb index 1280afbb095..2940d2bee6b 100644 --- a/lib/ddtrace/contrib/active_support/notifications/subscription.rb +++ b/lib/ddtrace/contrib/active_support/notifications/subscription.rb @@ -9,7 +9,8 @@ class Subscription attr_reader \ :tracer, :span_name, - :options + :options, + :block def initialize(tracer, span_name, options, &block) raise ArgumentError, 'Must be given a block!' unless block_given? @@ -25,12 +26,11 @@ def start(_name, _id, _payload) end def finish(name, id, payload) - span = tracer.active_span - - # The subscriber block needs to remember to set the name of the span. - @block.call(span, name, id, payload) - - span.finish + tracer.active_span.tap do |span| + return nil if span.nil? + block.call(span, name, id, payload) + span.finish + end end def subscribe(pattern) From fff1b7340052decd9e62ccdb5016339d05d68bbe Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 21 Mar 2018 15:46:18 -0400 Subject: [PATCH 33/72] Added: ActiveSupport specs to CI. --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index 375eab8b0df..6c54ae502e2 100644 --- a/Rakefile +++ b/Rakefile @@ -226,12 +226,14 @@ task :ci do sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake test:resque' # RSpec sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:active_record' + sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:active_support' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:dalli' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:faraday' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:graphql' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:racecar' sh 'rvm $MRI_VERSIONS --verbose do appraisal contrib rake spec:redis' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:active_record' + sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:active_support' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:dalli' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:faraday' sh 'rvm $MRI_OLD_VERSIONS --verbose do appraisal contrib-old rake spec:redis' From cce6a816e16681ca8e550ff1d7e166cf7b284d73 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 21 Mar 2018 16:15:47 -0400 Subject: [PATCH 34/72] Fixed: Load error with active_support/notifications. --- .../contrib/active_support/notifications/subscription.rb | 2 -- .../contrib/active_support/notifications/subscription_spec.rb | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/ddtrace/contrib/active_support/notifications/subscription.rb b/lib/ddtrace/contrib/active_support/notifications/subscription.rb index 2940d2bee6b..d460413126f 100644 --- a/lib/ddtrace/contrib/active_support/notifications/subscription.rb +++ b/lib/ddtrace/contrib/active_support/notifications/subscription.rb @@ -1,5 +1,3 @@ -require 'active_support/notifications' - module Datadog module Contrib module ActiveSupport diff --git a/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb b/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb index 470377d6810..00683239e43 100644 --- a/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb +++ b/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb @@ -1,6 +1,7 @@ require 'spec_helper' require 'ddtrace' +require 'active_support/notifications' require 'ddtrace/contrib/active_support/notifications/subscription' RSpec.describe Datadog::Contrib::ActiveSupport::Notifications::Subscription do From 6e5cc1db067b0ede553bf45ecb1f7d18acbb88af Mon Sep 17 00:00:00 2001 From: David Elner Date: Thu, 1 Mar 2018 18:05:37 -0500 Subject: [PATCH 35/72] Added: Elasticsearch body tag quantization. --- lib/ddtrace/contrib/elasticsearch/patcher.rb | 5 ++- lib/ddtrace/contrib/elasticsearch/quantize.rb | 45 +++++++++++++++++++ test/contrib/elasticsearch/quantize_test.rb | 22 +++++++++ test/contrib/elasticsearch/transport_test.rb | 4 +- 4 files changed, 73 insertions(+), 3 deletions(-) diff --git a/lib/ddtrace/contrib/elasticsearch/patcher.rb b/lib/ddtrace/contrib/elasticsearch/patcher.rb index 72061889a97..fbf699bf6b2 100644 --- a/lib/ddtrace/contrib/elasticsearch/patcher.rb +++ b/lib/ddtrace/contrib/elasticsearch/patcher.rb @@ -89,7 +89,10 @@ def perform_request(*args) span.set_tag(METHOD, method) span.set_tag(URL, url) span.set_tag(PARAMS, params) if params - span.set_tag(BODY, body) if body + if body + quantized_body = Datadog::Contrib::Elasticsearch::Quantize.format_body(body) + span.set_tag(BODY, quantized_body) + end span.set_tag('out.host', host) if host span.set_tag('out.port', port) if port diff --git a/lib/ddtrace/contrib/elasticsearch/quantize.rb b/lib/ddtrace/contrib/elasticsearch/quantize.rb index 51180958a93..01f00031252 100644 --- a/lib/ddtrace/contrib/elasticsearch/quantize.rb +++ b/lib/ddtrace/contrib/elasticsearch/quantize.rb @@ -3,6 +3,10 @@ module Contrib module Elasticsearch # Quantize contains ES-specific resource quantization tools. module Quantize + EXCLUDE_KEYS = [].freeze + SHOW_KEYS = [:_index, :_type, :_id].freeze + PLACEHOLDER = '?'.freeze + ID_REGEXP = %r{\/([0-9]+)([\/\?]|$)} ID_PLACEHOLDER = '/?\2'.freeze @@ -16,6 +20,47 @@ def format_url(url) quantized_url = url.gsub(ID_REGEXP, ID_PLACEHOLDER) quantized_url.gsub(INDEX_REGEXP, INDEX_PLACEHOLDER) end + + def format_body(body, exclude = [], show = []) + # Determine if bulk query or not, based on content + statements = body.end_with?("\n") ? body.split("\n") : [body] + + # Attempt to parse each + statements.collect do |s| + begin + JSON.dump(format_statement(JSON.parse(s), EXCLUDE_KEYS, SHOW_KEYS)) + rescue JSON::ParserError + # If it can't parse/dump, don't raise an error. + PLACEHOLDER + end + end.join("\n") + end + + def format_statement(statement, exclude = [], show = []) + case statement + when Hash + statement.each_with_object({}) do |(key, value), quantized| + if show == :all || show.include?(key.to_sym) + quantized[key] = value + elsif !exclude.include?(key.to_sym) + quantized[key] = format_value(value, exclude, show) + end + end + else + format_value(statement, exclude, show) + end + end + + def format_value(value, exclude = [], show = []) + case value + when Hash + format_statement(value, exclude, show) + when Array + format_value(value.first, exclude, show) + else + show == :all ? value : PLACEHOLDER + end + end end end end diff --git a/test/contrib/elasticsearch/quantize_test.rb b/test/contrib/elasticsearch/quantize_test.rb index d5bd5b6d073..de51c81dcc0 100644 --- a/test/contrib/elasticsearch/quantize_test.rb +++ b/test/contrib/elasticsearch/quantize_test.rb @@ -21,4 +21,26 @@ def test_index def test_combine assert_equal('/my?/thing/?', Datadog::Contrib::Elasticsearch::Quantize.format_url('/my123/thing/456789')) end + + def test_body + # MGet format + body = "{\"ids\":[\"1\",\"2\",\"3\"]}" + quantized_body = "{\"ids\":\"?\"}" + assert_equal(quantized_body, Datadog::Contrib::Elasticsearch::Quantize.format_body(body)) + + # Search format + body = "{\"query\":{\"match\":{\"title\":\"test\"}}}" + quantized_body = "{\"query\":{\"match\":{\"title\":\"?\"}}}" + assert_equal(quantized_body, Datadog::Contrib::Elasticsearch::Quantize.format_body(body)) + + # MSearch format + body = "{}\n{\"query\":{\"match_all\":{}}}\n{\"index\":\"myindex\",\"type\":\"mytype\"}\n{\"query\":{\"query_string\":{\"query\":\"\\\"test\\\"\"}}}\n{\"search_type\":\"count\"}\n{\"aggregations\":{\"published\":{\"terms\":{\"field\":\"published\"}}}}\n" + quantized_body = "{}\n{\"query\":{\"match_all\":{}}}\n{\"index\":\"?\",\"type\":\"?\"}\n{\"query\":{\"query_string\":{\"query\":\"?\"}}}\n{\"search_type\":\"?\"}\n{\"aggregations\":{\"published\":{\"terms\":{\"field\":\"?\"}}}}" + assert_equal(quantized_body, Datadog::Contrib::Elasticsearch::Quantize.format_body(body)) + + # Bulk format + body = "{\"index\":{\"_index\":\"myindex\",\"_type\":\"mytype\",\"_id\":1}}\n{\"title\":\"foo\"}\n{\"index\":{\"_index\":\"myindex\",\"_type\":\"mytype\",\"_id\":2}}\n{\"title\":\"foo\"}\n" + quantized_body = "{\"index\":{\"_index\":\"myindex\",\"_type\":\"mytype\",\"_id\":1}}\n{\"title\":\"?\"}\n{\"index\":{\"_index\":\"myindex\",\"_type\":\"mytype\",\"_id\":2}}\n{\"title\":\"?\"}" + assert_equal(quantized_body, Datadog::Contrib::Elasticsearch::Quantize.format_body(body)) + end end diff --git a/test/contrib/elasticsearch/transport_test.rb b/test/contrib/elasticsearch/transport_test.rb index f7e70255c51..59e5c4b617c 100644 --- a/test/contrib/elasticsearch/transport_test.rb +++ b/test/contrib/elasticsearch/transport_test.rb @@ -55,7 +55,7 @@ def test_perform_request_with_encoded_body assert_equal('PUT', span.get_tag('elasticsearch.method')) assert_equal('201', span.get_tag('http.status_code')) assert_equal("{\"refresh\":true\}", span.get_tag('elasticsearch.params')) - assert_equal('{"data1":"D1","data2":"D2"}', span.get_tag('elasticsearch.body')) + assert_equal('{"data1":"?","data2":"?"}', span.get_tag('elasticsearch.body')) end def roundtrip_put @@ -71,7 +71,7 @@ def roundtrip_put assert_equal('PUT', span.get_tag('elasticsearch.method')) assert_equal('201', span.get_tag('http.status_code')) assert_equal("{\"refresh\":true\}", span.get_tag('elasticsearch.params')) - assert_equal('{"data1":"D1","data2":"D2"}', span.get_tag('elasticsearch.body')) + assert_equal('{"data1":"?","data2":"?"}', span.get_tag('elasticsearch.body')) end def roundtrip_get From a39d0076d76b3c1218076516416b36828592e7df Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 2 Mar 2018 11:27:46 -0500 Subject: [PATCH 36/72] Changed: Allow options to be passed to ElasticSearch quantization. --- lib/ddtrace/contrib/elasticsearch/quantize.rb | 74 ++++++++++++++----- test/contrib/elasticsearch/quantize_test.rb | 26 +++++++ 2 files changed, 82 insertions(+), 18 deletions(-) diff --git a/lib/ddtrace/contrib/elasticsearch/quantize.rb b/lib/ddtrace/contrib/elasticsearch/quantize.rb index 01f00031252..59e256335f0 100644 --- a/lib/ddtrace/contrib/elasticsearch/quantize.rb +++ b/lib/ddtrace/contrib/elasticsearch/quantize.rb @@ -3,9 +3,10 @@ module Contrib module Elasticsearch # Quantize contains ES-specific resource quantization tools. module Quantize + PLACEHOLDER = '?'.freeze EXCLUDE_KEYS = [].freeze SHOW_KEYS = [:_index, :_type, :_id].freeze - PLACEHOLDER = '?'.freeze + DEFAULT_OPTIONS = { exclude: EXCLUDE_KEYS, show: SHOW_KEYS }.freeze ID_REGEXP = %r{\/([0-9]+)([\/\?]|$)} ID_PLACEHOLDER = '/?\2'.freeze @@ -21,44 +22,81 @@ def format_url(url) quantized_url.gsub(INDEX_REGEXP, INDEX_PLACEHOLDER) end - def format_body(body, exclude = [], show = []) + def format_body(body, options = {}) + options = merge_options(DEFAULT_OPTIONS, options) + # Determine if bulk query or not, based on content statements = body.end_with?("\n") ? body.split("\n") : [body] - # Attempt to parse each - statements.collect do |s| - begin - JSON.dump(format_statement(JSON.parse(s), EXCLUDE_KEYS, SHOW_KEYS)) - rescue JSON::ParserError - # If it can't parse/dump, don't raise an error. - PLACEHOLDER + # Parse each statement and quantize them. + statements.collect do |string| + reserialize_json(string) do |obj| + format_statement(obj, options) end end.join("\n") end - def format_statement(statement, exclude = [], show = []) + def format_statement(statement, options = {}) + return statement if options[:show] == :all + case statement when Hash statement.each_with_object({}) do |(key, value), quantized| - if show == :all || show.include?(key.to_sym) + if options[:show].include?(key.to_sym) quantized[key] = value - elsif !exclude.include?(key.to_sym) - quantized[key] = format_value(value, exclude, show) + elsif !options[:exclude].include?(key.to_sym) + quantized[key] = format_value(value, options) end end else - format_value(statement, exclude, show) + format_value(statement, options) end end - def format_value(value, exclude = [], show = []) + def format_value(value, options = {}) + return value if options[:show] == :all + case value when Hash - format_statement(value, exclude, show) + format_statement(value, options) when Array - format_value(value.first, exclude, show) + # If any are objects, format them. + if value.any? { |v| v.class <= Hash || v.class <= Array } + value.collect { |i| format_value(i, options) } + # Otherwise short-circuit and return single placeholder + else + PLACEHOLDER + end else - show == :all ? value : PLACEHOLDER + PLACEHOLDER + end + end + + def merge_options(original, additional) + {}.tap do |options| + # Show + # If either is :all, value becomes :all + options[:show] = if original[:show] == :all || additional[:show] == :all + :all + else + (original[:show] || []).dup.concat(additional[:show] || []).uniq + end + + # Exclude + options[:exclude] = (original[:exclude] || []).dup.concat(additional[:exclude] || []).uniq + end + end + + # Parses a JSON object from a string, passes its value + # to the block provided, and dumps its result back to JSON. + # If JSON parsing fails, it prints fail_value. + def reserialize_json(string, fail_value = PLACEHOLDER) + return string unless block_given? + begin + JSON.dump(yield(JSON.parse(string))) + rescue JSON::ParserError + # If it can't parse/dump, don't raise an error. + fail_value end end end diff --git a/test/contrib/elasticsearch/quantize_test.rb b/test/contrib/elasticsearch/quantize_test.rb index de51c81dcc0..8c51004b11d 100644 --- a/test/contrib/elasticsearch/quantize_test.rb +++ b/test/contrib/elasticsearch/quantize_test.rb @@ -22,6 +22,8 @@ def test_combine assert_equal('/my?/thing/?', Datadog::Contrib::Elasticsearch::Quantize.format_url('/my123/thing/456789')) end + # rubocop:disable Metrics/LineLength + # rubocop:disable Style/StringLiterals def test_body # MGet format body = "{\"ids\":[\"1\",\"2\",\"3\"]}" @@ -43,4 +45,28 @@ def test_body quantized_body = "{\"index\":{\"_index\":\"myindex\",\"_type\":\"mytype\",\"_id\":1}}\n{\"title\":\"?\"}\n{\"index\":{\"_index\":\"myindex\",\"_type\":\"mytype\",\"_id\":2}}\n{\"title\":\"?\"}" assert_equal(quantized_body, Datadog::Contrib::Elasticsearch::Quantize.format_body(body)) end + + def test_body_show + body = "{\"query\":{\"match\":{\"title\":\"test\",\"subtitle\":\"test\"}}}" + quantized_body = "{\"query\":{\"match\":{\"title\":\"test\",\"subtitle\":\"?\"}}}" + assert_equal(quantized_body, Datadog::Contrib::Elasticsearch::Quantize.format_body(body, show: [:title])) + + body = "{\"query\":{\"match\":{\"title\":\"test\",\"subtitle\":\"test\"}}}" + quantized_body = "{\"query\":{\"match\":{\"title\":\"test\",\"subtitle\":\"test\"}}}" + assert_equal(quantized_body, Datadog::Contrib::Elasticsearch::Quantize.format_body(body, show: :all)) + + body = "[{\"foo\":\"foo\"},{\"bar\":\"bar\"}]" + quantized_body = "[{\"foo\":\"foo\"},{\"bar\":\"bar\"}]" + assert_equal(quantized_body, Datadog::Contrib::Elasticsearch::Quantize.format_body(body, show: :all)) + + body = "[\"foo\",\"bar\"]" + quantized_body = "[\"foo\",\"bar\"]" + assert_equal(quantized_body, Datadog::Contrib::Elasticsearch::Quantize.format_body(body, show: :all)) + end + + def test_body_exclude + body = "{\"query\":{\"match\":{\"title\":\"test\",\"subtitle\":\"test\"}}}" + quantized_body = "{\"query\":{\"match\":{\"subtitle\":\"?\"}}}" + assert_equal(quantized_body, Datadog::Contrib::Elasticsearch::Quantize.format_body(body, exclude: [:title])) + end end From 64546f60dfdcc4b9de8bc3ef08156e1516404b24 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 2 Mar 2018 11:41:38 -0500 Subject: [PATCH 37/72] Added: quantize option for ElasticSearch. --- docs/GettingStarted.md | 1 + lib/ddtrace/contrib/elasticsearch/patcher.rb | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index 89957cd98ea..fac86c65f81 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -274,6 +274,7 @@ Where `options` is an optional `Hash` that accepts the following parameters: | Key | Description | Default | | --- | --- | --- | | ``service_name`` | Service name used for `elasticsearch` instrumentation | elasticsearch | +| ``quantize`` | Hash containing options for quantization. May include `:show` with an Array of keys to not quantize (or `:all` to skip quantization), or `:exclude` with Array of keys to exclude entirely. | {} | ### MongoDB diff --git a/lib/ddtrace/contrib/elasticsearch/patcher.rb b/lib/ddtrace/contrib/elasticsearch/patcher.rb index fbf699bf6b2..fdc327d181c 100644 --- a/lib/ddtrace/contrib/elasticsearch/patcher.rb +++ b/lib/ddtrace/contrib/elasticsearch/patcher.rb @@ -15,6 +15,7 @@ module Patcher include Base register_as :elasticsearch, auto_patch: true option :service_name, default: SERVICE + option :quantize, default: {} @patched = false @@ -90,7 +91,8 @@ def perform_request(*args) span.set_tag(URL, url) span.set_tag(PARAMS, params) if params if body - quantized_body = Datadog::Contrib::Elasticsearch::Quantize.format_body(body) + quantize_options = Datadog.configuration[:elasticsearch][:quantize] + quantized_body = Datadog::Contrib::Elasticsearch::Quantize.format_body(body, quantize_options) span.set_tag(BODY, quantized_body) end span.set_tag('out.host', host) if host From 7828c76c6ee66e58025c0f64e6e433ac360a15df Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 26 Mar 2018 05:21:22 -0400 Subject: [PATCH 38/72] Racecar integration to use ActiveSupport::Notifications::Subscriber (#381) Refactoring Racecar integration after #380 --- lib/ddtrace/contrib/racecar/patcher.rb | 35 +++++++------------- spec/ddtrace/contrib/racecar/patcher_spec.rb | 8 +++++ 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/lib/ddtrace/contrib/racecar/patcher.rb b/lib/ddtrace/contrib/racecar/patcher.rb index 5da2a489531..a26d128501b 100644 --- a/lib/ddtrace/contrib/racecar/patcher.rb +++ b/lib/ddtrace/contrib/racecar/patcher.rb @@ -1,4 +1,5 @@ require 'ddtrace/ext/app_types' +require 'ddtrace/contrib/active_support/notifications/subscriber' module Datadog module Contrib @@ -6,19 +7,24 @@ module Racecar # Provides instrumentation for `racecar` through ActiveSupport instrumentation signals module Patcher include Base + include ActiveSupport::Notifications::Subscriber + NAME_MESSAGE = 'racecar.message'.freeze NAME_BATCH = 'racecar.batch'.freeze register_as :racecar option :tracer, default: Datadog.tracer option :service_name, default: 'racecar' + on_subscribe do + subscribe('process_message.racecar', self::NAME_MESSAGE, {}, configuration[:tracer], &method(:process)) + subscribe('process_batch.racecar', self::NAME_BATCH, {}, configuration[:tracer], &method(:process)) + end + class << self def patch return patched? if patched? || !compatible? - ::ActiveSupport::Notifications.subscribe('process_batch.racecar', self) - ::ActiveSupport::Notifications.subscribe('process_message.racecar', self) - + subscribe! configuration[:tracer].set_service_info( configuration[:service_name], 'racecar', @@ -33,28 +39,17 @@ def patched? @patched = false end - def start(event, _, payload) - ensure_clean_context! - - name = event[/message/] ? NAME_MESSAGE : NAME_BATCH - span = configuration[:tracer].trace(name) + def process(span, event, _, payload) span.service = configuration[:service_name] span.resource = payload[:consumer_class] + span.set_tag('kafka.topic', payload[:topic]) span.set_tag('kafka.consumer', payload[:consumer_class]) span.set_tag('kafka.partition', payload[:partition]) span.set_tag('kafka.offset', payload[:offset]) if payload.key?(:offset) span.set_tag('kafka.first_offset', payload[:first_offset]) if payload.key?(:first_offset) span.set_tag('kafka.message_count', payload[:message_count]) if payload.key?(:message_count) - end - - def finish(_, _, payload) - current_span = configuration[:tracer].call_context.current_span - - return unless current_span - - current_span.set_error(payload[:exception_object]) if payload[:exception_object] - current_span.finish + span.set_error(payload[:exception_object]) if payload[:exception_object] end private @@ -66,12 +61,6 @@ def configuration def compatible? defined?(::Racecar) && defined?(::ActiveSupport::Notifications) end - - def ensure_clean_context! - return unless configuration[:tracer].call_context.current_span - - configuration[:tracer].provider.context = Context.new - end end end end diff --git a/spec/ddtrace/contrib/racecar/patcher_spec.rb b/spec/ddtrace/contrib/racecar/patcher_spec.rb index bba4ca1c5fb..7e219836f9f 100644 --- a/spec/ddtrace/contrib/racecar/patcher_spec.rb +++ b/spec/ddtrace/contrib/racecar/patcher_spec.rb @@ -17,6 +17,14 @@ def all_spans Datadog.configure do |c| c.use :racecar, tracer: tracer end + + # Make sure to update the subscription tracer, + # so we aren't writing to a stale tracer. + if Datadog::Contrib::Racecar::Patcher.patched? + Datadog::Contrib::Racecar::Patcher.subscriptions.each do |subscription| + allow(subscription).to receive(:tracer).and_return(tracer) + end + end end describe 'for single message processing' do From a141ce029ee8e38ac8a3239b5a6ebe2e6fddcbd5 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 26 Mar 2018 16:31:58 -0400 Subject: [PATCH 39/72] Added: Datadog::Quantization::HTTP module for quantizing HTTP resources. --- lib/ddtrace.rb | 1 + lib/ddtrace/quantization/http.rb | 86 ++++++++++ spec/ddtrace/quantization/http_spec.rb | 227 +++++++++++++++++++++++++ 3 files changed, 314 insertions(+) create mode 100644 lib/ddtrace/quantization/http.rb create mode 100644 spec/ddtrace/quantization/http_spec.rb diff --git a/lib/ddtrace.rb b/lib/ddtrace.rb index 664eab25d8d..eb213184e2a 100644 --- a/lib/ddtrace.rb +++ b/lib/ddtrace.rb @@ -4,6 +4,7 @@ require 'ddtrace/pin' require 'ddtrace/tracer' require 'ddtrace/error' +require 'ddtrace/quantization/http' require 'ddtrace/pipeline' require 'ddtrace/configuration' require 'ddtrace/patcher' diff --git a/lib/ddtrace/quantization/http.rb b/lib/ddtrace/quantization/http.rb new file mode 100644 index 00000000000..48ba93a5542 --- /dev/null +++ b/lib/ddtrace/quantization/http.rb @@ -0,0 +1,86 @@ +require 'uri' +require 'set' + +module Datadog + module Quantization + # Quantization for HTTP resources + module HTTP + PLACEHOLDER = '?'.freeze + + module_function + + def url(url, options = {}) + url!(url, options) + rescue StandardError + options[:placeholder] || PLACEHOLDER + end + + def url!(url, options = {}) + options ||= {} + + URI.parse(url).tap do |uri| + # Format the query string + if uri.query + query = query(uri.query, options[:query]) + uri.query = (!query.nil? && query.empty? ? nil : query) + end + + # Remove any URI framents + uri.fragment = nil unless options[:fragment] == :show + end.to_s + end + + def query(query, options = {}) + query!(query, options) + rescue StandardError + options[:placeholder] || PLACEHOLDER + end + + def query!(query, options = {}) + options ||= {} + options[:show] = options[:show] || [] + options[:exclude] = options[:exclude] || [] + + # Short circuit if query string is meant to exclude everything + # or if the query string is meant to include everything + return '' if options[:exclude] == :all + return query if options[:show] == :all + + collect_query(query, uniq: true) do |key, value| + if options[:exclude].include?(key) + [nil, nil] + else + value = options[:show].include?(key) ? value : nil + [key, value] + end + end + end + + # Iterate over each key value pair, yielding to the block given. + # Accepts :uniq option, which keeps uniq copies of keys without values. + # e.g. Reduces "foo&bar=bar&bar=bar&foo" to "foo&bar=bar&bar=bar" + def collect_query(query, options = {}) + return query unless block_given? + uniq = options[:uniq].nil? ? false : options[:uniq] + keys = Set.new + + delims = query.scan(/(^|&|;)/).flatten + query.split(/[&;]/).collect.with_index do |pairs, i| + key, value = pairs.split('=', 2) + key, value = yield(key, value, delims[i]) + if uniq && keys.include?(key) + '' + elsif key && value + "#{delims[i]}#{key}=#{value}" + elsif key + "#{delims[i]}#{key}".tap { keys << key } + else + '' + end + end.join.sub(/^[&;]/, '') + end + + private_class_method :collect_query + end + end +end diff --git a/spec/ddtrace/quantization/http_spec.rb b/spec/ddtrace/quantization/http_spec.rb new file mode 100644 index 00000000000..24db2172c23 --- /dev/null +++ b/spec/ddtrace/quantization/http_spec.rb @@ -0,0 +1,227 @@ +require 'spec_helper' + +require 'ddtrace/quantization/http' + +RSpec.describe Datadog::Quantization::HTTP do + describe '#url' do + subject(:result) { described_class.url(url, options) } + let(:options) { {} } + + context 'given a URL' do + let(:url) { 'http://example.com/path?category_id=1&sort_by=asc#featured' } + + context 'default behavior' do + it { is_expected.to eq('http://example.com/path?category_id&sort_by') } + end + + context 'default behavior for an array' do + let(:url) { 'http://example.com/path?categories[]=1&categories[]=2' } + it { is_expected.to eq('http://example.com/path?categories[]') } + end + + context 'with query: show: value' do + let(:options) { { query: { show: ['category_id'] } } } + it { is_expected.to eq('http://example.com/path?category_id=1&sort_by') } + end + + context 'with query: show: :all' do + let(:options) { { query: { show: :all } } } + it { is_expected.to eq('http://example.com/path?category_id=1&sort_by=asc') } + end + + context 'with query: exclude: value' do + let(:options) { { query: { exclude: ['sort_by'] } } } + it { is_expected.to eq('http://example.com/path?category_id') } + end + + context 'with query: exclude: :all' do + let(:options) { { query: { exclude: :all } } } + it { is_expected.to eq('http://example.com/path') } + end + + context 'with show: :all' do + let(:options) { { fragment: :show } } + it { is_expected.to eq('http://example.com/path?category_id&sort_by#featured') } + end + + context 'with Unicode characters' do + # URLs do not permit unencoded non-ASCII characters in the URL. + let(:url) { 'http://example.com/path?繋がってて' } + it { is_expected.to eq(described_class::PLACEHOLDER) } + end + end + end + + describe '#query' do + subject(:result) { described_class.query(query, options) } + + context 'given a query' do + context 'and no options' do + let(:options) { {} } + + context 'with a single parameter' do + let(:query) { 'foo=foo' } + it { is_expected.to eq('foo') } + + context 'with an invalid byte sequence' do + # \255 is off-limits https://en.wikipedia.org/wiki/UTF-8#Codepage_layout + # There isn't a graceful way to handle this without stripping interesting + # characters out either; so just raise an error and default to the placeholder. + let(:query) { "foo\255=foo" } + it { is_expected.to eq('?') } + end + end + + context 'with multiple parameters' do + let(:query) { 'foo=foo&bar=bar' } + it { is_expected.to eq('foo&bar') } + end + + context 'with array-style parameters' do + let(:query) { 'foo[]=bar&foo[]=baz' } + it { is_expected.to eq('foo[]') } + end + + context 'with semi-colon style parameters' do + let(:query) { 'foo;bar' } + # Notice semicolons aren't preseved... no great way of handling this. + # Semicolons are illegal as of 2014... so this is an edge case. + # See https://www.w3.org/TR/2014/REC-html5-20141028/forms.html#url-encoded-form-data + it { is_expected.to eq('foo;bar') } + end + + context 'with object-style parameters' do + let(:query) { 'user[id]=1&user[name]=Nathan' } + it { is_expected.to eq('user[id]&user[name]') } + + context 'that are complex' do + let(:query) { 'users[][id]=1&users[][name]=Nathan&users[][id]=2&users[][name]=Emma' } + it { is_expected.to eq('users[][id]&users[][name]') } + end + end + end + + context 'and a show: :all option' do + let(:query) { 'foo=foo&bar=bar' } + let(:options) { { show: :all } } + it { is_expected.to eq(query) } + end + + context 'and a show option' do + context 'with a single parameter' do + let(:query) { 'foo=foo' } + let(:key) { 'foo' } + let(:options) { { show: [key] } } + it { is_expected.to eq('foo=foo') } + + context 'that has a Unicode key' do + let(:query) { '繋=foo' } + let(:key) { '繋' } + it { is_expected.to eq('繋=foo') } + + context 'that is encoded' do + let(:query) { '%E7%B9%8B=foo' } + let(:key) { '%E7%B9%8B' } + it { is_expected.to eq('%E7%B9%8B=foo') } + end + end + + context 'that has a Unicode value' do + let(:query) { 'foo=繋' } + let(:key) { 'foo' } + it { is_expected.to eq('foo=繋') } + + context 'that is encoded' do + let(:query) { 'foo=%E7%B9%8B' } + it { is_expected.to eq('foo=%E7%B9%8B') } + end + end + + context 'that has a Unicode key and value' do + let(:query) { '繋=繋' } + let(:key) { '繋' } + it { is_expected.to eq('繋=繋') } + + context 'that is encoded' do + let(:query) { '%E7%B9%8B=%E7%B9%8B' } + let(:key) { '%E7%B9%8B' } + it { is_expected.to eq('%E7%B9%8B=%E7%B9%8B') } + end + end + end + + context 'with multiple parameters' do + let(:query) { 'foo=foo&bar=bar' } + let(:options) { { show: ['foo'] } } + it { is_expected.to eq('foo=foo&bar') } + end + + context 'with array-style parameters' do + let(:query) { 'foo[]=bar&foo[]=baz' } + let(:options) { { show: ['foo[]'] } } + it { is_expected.to eq('foo[]=bar&foo[]=baz') } + + context 'that contains encoded braces' do + let(:query) { 'foo[]=%5Bbar%5D&foo[]=%5Bbaz%5D' } + it { is_expected.to eq('foo[]=%5Bbar%5D&foo[]=%5Bbaz%5D') } + + context 'that exactly matches the key' do + let(:query) { 'foo[]=foo%5B%5D&foo[]=foo%5B%5D' } + it { is_expected.to eq('foo[]=foo%5B%5D&foo[]=foo%5B%5D') } + end + end + end + + context 'with object-style parameters' do + let(:query) { 'user[id]=1&user[name]=Nathan' } + let(:options) { { show: ['user[id]'] } } + it { is_expected.to eq('user[id]=1&user[name]') } + + context 'that are complex' do + let(:query) { 'users[][id]=1&users[][name]=Nathan&users[][id]=2&users[][name]=Emma' } + let(:options) { { show: ['users[][id]'] } } + it { is_expected.to eq('users[][id]=1&users[][name]&users[][id]=2') } + end + end + end + + context 'and an exclude: :all option' do + let(:query) { 'foo=foo&bar=bar' } + let(:options) { { exclude: :all } } + it { is_expected.to eq('') } + end + + context 'and an exclude option' do + context 'with a single parameter' do + let(:query) { 'foo=foo' } + let(:options) { { exclude: ['foo'] } } + it { is_expected.to eq('') } + end + + context 'with multiple parameters' do + let(:query) { 'foo=foo&bar=bar' } + let(:options) { { exclude: ['foo'] } } + it { is_expected.to eq('bar') } + end + + context 'with array-style parameters' do + let(:query) { 'foo[]=bar&foo[]=baz' } + let(:options) { { exclude: ['foo[]'] } } + it { is_expected.to eq('') } + end + + context 'with object-style parameters' do + let(:query) { 'user[id]=1&user[name]=Nathan' } + let(:options) { { exclude: ['user[name]'] } } + it { is_expected.to eq('user[id]') } + + context 'that are complex' do + let(:query) { 'users[][id]=1&users[][name]=Nathan&users[][id]=2&users[][name]=Emma' } + let(:options) { { exclude: ['users[][name]'] } } + it { is_expected.to eq('users[][id]') } + end + end + end + end + end +end From 3204d8a28d94b4267bfb52cdd393302c5c2d0af5 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 26 Mar 2018 16:54:09 -0400 Subject: [PATCH 40/72] Fixed: Quantizer spec UTF-8 string incompatibility with Ruby 1.9.3 --- spec/ddtrace/quantization/http_spec.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/ddtrace/quantization/http_spec.rb b/spec/ddtrace/quantization/http_spec.rb index 24db2172c23..e6c3aa0e36c 100644 --- a/spec/ddtrace/quantization/http_spec.rb +++ b/spec/ddtrace/quantization/http_spec.rb @@ -1,3 +1,4 @@ +# encoding: utf-8 require 'spec_helper' require 'ddtrace/quantization/http' @@ -46,7 +47,7 @@ context 'with Unicode characters' do # URLs do not permit unencoded non-ASCII characters in the URL. - let(:url) { 'http://example.com/path?繋がってて' } + let(:url) { "http://example.com/path?繋がってて" } it { is_expected.to eq(described_class::PLACEHOLDER) } end end From c6f342f71303ba2f39014f9dd63063bd4c3bba44 Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 27 Mar 2018 11:06:02 -0400 Subject: [PATCH 41/72] Added: Error handling to ElasticSearch quantizer. --- lib/ddtrace/contrib/elasticsearch/quantize.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/ddtrace/contrib/elasticsearch/quantize.rb b/lib/ddtrace/contrib/elasticsearch/quantize.rb index 59e256335f0..1f9d66ce997 100644 --- a/lib/ddtrace/contrib/elasticsearch/quantize.rb +++ b/lib/ddtrace/contrib/elasticsearch/quantize.rb @@ -23,6 +23,12 @@ def format_url(url) end def format_body(body, options = {}) + format_body!(body, options) + rescue StandardError + PLACEHOLDER + end + + def format_body!(body, options = {}) options = merge_options(DEFAULT_OPTIONS, options) # Determine if bulk query or not, based on content From dacba60a25086142e1cb144b3ee2b1b27195dac5 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 26 Mar 2018 17:29:22 -0400 Subject: [PATCH 42/72] Added: URL quantization to Rack. --- docs/GettingStarted.md | 35 +++++++++++++++++++++++++ lib/ddtrace/contrib/rack/middlewares.rb | 3 ++- lib/ddtrace/contrib/rack/patcher.rb | 1 + test/contrib/rack/middleware_test.rb | 29 ++++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index 89957cd98ea..d6d9f5fdcae 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -142,9 +142,44 @@ Where `options` is an optional `Hash` that accepts the following parameters: | ``service_name`` | Service name used when tracing application requests | rack | | ``distributed_tracing`` | Enables [distributed tracing](#Distributed_Tracing) so that this service trace is connected with a trace of another service if tracing headers are received | `false` | | ``middleware_names`` | Enable this if you want to use the middleware classes as the resource names for `rack` spans. Must provide the ``application`` option with it. | ``false`` | +| ``quantize`` | Hash containing options for quantization. May include `:query` or `:fragment`. | {} | +| ``quantize.query`` | Hash containing options for query portion of URL quantization. May include `:show` or `:exclude`. See options below. Option must be nested inside the `quantize` option. | {} | +| ``quantize.query.show`` | Defines which values should always be shown. Shows no values by default. May be an Array of strings, or `:all` to show all values. Option must be nested inside the `query` option. | ``nil`` | +| ``quantize.query.exclude`` | Defines which values should be removed entirely. Excludes nothing by default. May be an Array of strings, or `:all` to remove the query string entirely. Option must be nested inside the `query` option. | ``nil`` | +| ``quantize.fragment`` | Defines behavior for URL fragments. Removes fragments by default. May be `:show` to show URL fragments. Option must be nested inside the `quantize` option. | ``nil`` | | ``application`` | Your Rack application. Necessary for enabling middleware resource names. | ``nil`` | | ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | +Configuring URL quantization behavior: + +``` +Datadog.configure do |c| + # Default behavior: all values are quantized, fragment is removed. + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id&sort_by + # http://example.com/path?categories[]=1&categories[]=2 --> http://example.com/path?categories[] + + # Show values for any query string parameter matching 'category_id' exactly + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id=1&sort_by + c.use :rack, quantize: { query: { show: ['category_id'] } } + + # Show all values for all query string parameters + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id=1&sort_by=asc + c.use :rack, quantize: { query: { show: :all } } + + # Totally exclude any query string parameter matching 'sort_by' exactly + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id + c.use :rack, quantize: { query: { exclude: ['sort_by'] } } + + # Remove the query string entirely + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path + c.use :rack, quantize: { query: { exclude: :all } } + + # Show URL fragments + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id&sort_by#featured + c.use :rack, quantize: { fragment: :show } +end +``` + ## Other libraries ### GraphQL diff --git a/lib/ddtrace/contrib/rack/middlewares.rb b/lib/ddtrace/contrib/rack/middlewares.rb index 0818e3c1145..0b05d0242bc 100644 --- a/lib/ddtrace/contrib/rack/middlewares.rb +++ b/lib/ddtrace/contrib/rack/middlewares.rb @@ -104,7 +104,8 @@ def set_request_tags!(request_span, env, status, headers, response, original_env request_span.set_tag(Datadog::Ext::HTTP::METHOD, env['REQUEST_METHOD']) end if request_span.get_tag(Datadog::Ext::HTTP::URL).nil? - request_span.set_tag(Datadog::Ext::HTTP::URL, url) + options = Datadog.configuration[:rack][:quantize] + request_span.set_tag(Datadog::Ext::HTTP::URL, Datadog::Quantization::HTTP.url(url, options)) end if request_span.get_tag(Datadog::Ext::HTTP::BASE_URL).nil? request_obj = ::Rack::Request.new(env) diff --git a/lib/ddtrace/contrib/rack/patcher.rb b/lib/ddtrace/contrib/rack/patcher.rb index cc466e0dc10..50649916be1 100644 --- a/lib/ddtrace/contrib/rack/patcher.rb +++ b/lib/ddtrace/contrib/rack/patcher.rb @@ -8,6 +8,7 @@ module Patcher option :tracer, default: Datadog.tracer option :distributed_tracing, default: false option :middleware_names, default: false + option :quantize, default: {} option :application option :service_name, default: 'rack', depends_on: [:tracer] do |value| get_option(:tracer).set_service_info(value, 'rack', Ext::AppTypes::WEB) diff --git a/test/contrib/rack/middleware_test.rb b/test/contrib/rack/middleware_test.rb index 521757f2cfa..c8221a7ad7a 100644 --- a/test/contrib/rack/middleware_test.rb +++ b/test/contrib/rack/middleware_test.rb @@ -64,6 +64,35 @@ def test_request_middleware_get_with_request_uri assert_equal('200', span.get_tag('http.status_code')) # Since REQUEST_URI is set (usually provided by WEBrick/Puma) # it uses REQUEST_URI, which has query string parameters. + # However, that query string will be quantized. + assert_equal('/success?foo', span.get_tag('http.url')) + assert_equal('http://example.org', span.get_tag('http.base_url')) + assert_equal(0, span.status) + assert_nil(span.parent) + end + + def test_request_middleware_get_with_request_uri_and_quantize_option + Datadog.configure do |c| + c.use :rack, quantize: { query: { show: ['foo'] } } + end + + # ensure the Rack request is properly traced + get '/success?foo=bar', {}, 'REQUEST_URI' => '/success?foo=bar' + assert last_response.ok? + + spans = @tracer.writer.spans + assert_equal(1, spans.length) + + span = spans[0] + assert_equal('rack.request', span.name) + assert_equal('http', span.span_type) + assert_equal('rack', span.service) + assert_equal('GET 200', span.resource) + assert_equal('GET', span.get_tag('http.method')) + assert_equal('200', span.get_tag('http.status_code')) + # Since REQUEST_URI is set (usually provided by WEBrick/Puma) + # it uses REQUEST_URI, which has query string parameters. + # However, that query string will be quantized. assert_equal('/success?foo=bar', span.get_tag('http.url')) assert_equal('http://example.org', span.get_tag('http.base_url')) assert_equal(0, span.status) From d6ef28340b10252f75a21e4b9508fc7734d6b357 Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 27 Mar 2018 15:29:24 -0400 Subject: [PATCH 43/72] Changed: rails5 appraisal to use Rails 5.1 instead of Rails 5.0. --- Appraisals | 8 ++++---- gemfiles/rails5_mysql2.gemfile | 2 +- gemfiles/rails5_postgres.gemfile | 2 +- gemfiles/rails5_postgres_redis.gemfile | 2 +- gemfiles/rails5_postgres_sidekiq.gemfile | 2 +- test/contrib/rails/apps/application.rb | 1 - test/contrib/rails/test_helper.rb | 2 +- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Appraisals b/Appraisals index e4074051e6d..636b01865b5 100644 --- a/Appraisals +++ b/Appraisals @@ -78,24 +78,24 @@ if RUBY_VERSION < '2.4.0' && RUBY_PLATFORM != 'java' if RUBY_VERSION >= '2.2.2' appraise 'rails5-mysql2' do - gem 'rails', '5.0.1' + gem 'rails', '5.1.5' gem 'mysql2', '< 0.5', platform: :ruby end appraise 'rails5-postgres' do - gem 'rails', '5.0.1' + gem 'rails', '5.1.5' gem 'pg', '< 1.0', platform: :ruby end appraise 'rails5-postgres-redis' do - gem 'rails', '5.0.1' + gem 'rails', '5.1.5' gem 'pg', '< 1.0', platform: :ruby gem 'redis-rails' gem 'redis' end appraise 'rails5-postgres-sidekiq' do - gem 'rails', '5.0.1' + gem 'rails', '5.1.5' gem 'pg', '< 1.0', platform: :ruby gem 'sidekiq' gem 'activejob' diff --git a/gemfiles/rails5_mysql2.gemfile b/gemfiles/rails5_mysql2.gemfile index cca2c942fa9..ca89bd2801e 100644 --- a/gemfiles/rails5_mysql2.gemfile +++ b/gemfiles/rails5_mysql2.gemfile @@ -3,7 +3,7 @@ source "https://rubygems.org" gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master" -gem "rails", "5.0.1" +gem "rails", "5.1.5" gem "mysql2", "< 0.5", platform: :ruby gemspec path: "../" diff --git a/gemfiles/rails5_postgres.gemfile b/gemfiles/rails5_postgres.gemfile index 2e7cdc518c4..116646789ee 100644 --- a/gemfiles/rails5_postgres.gemfile +++ b/gemfiles/rails5_postgres.gemfile @@ -3,7 +3,7 @@ source "https://rubygems.org" gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master" -gem "rails", "5.0.1" +gem "rails", "5.1.5" gem "pg", "< 1.0", platform: :ruby gemspec path: "../" diff --git a/gemfiles/rails5_postgres_redis.gemfile b/gemfiles/rails5_postgres_redis.gemfile index 3cf0bbb179b..dd95d68c7ca 100644 --- a/gemfiles/rails5_postgres_redis.gemfile +++ b/gemfiles/rails5_postgres_redis.gemfile @@ -3,7 +3,7 @@ source "https://rubygems.org" gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master" -gem "rails", "5.0.1" +gem "rails", "5.1.5" gem "pg", "< 1.0", platform: :ruby gem "redis-rails" gem "redis" diff --git a/gemfiles/rails5_postgres_sidekiq.gemfile b/gemfiles/rails5_postgres_sidekiq.gemfile index 87c84ae1e4f..86913ffb2ed 100644 --- a/gemfiles/rails5_postgres_sidekiq.gemfile +++ b/gemfiles/rails5_postgres_sidekiq.gemfile @@ -3,7 +3,7 @@ source "https://rubygems.org" gem "pry-nav", git: "https://github.com/nixme/pry-nav.git", branch: "master" -gem "rails", "5.0.1" +gem "rails", "5.1.5" gem "pg", "< 1.0", platform: :ruby gem "sidekiq" gem "activejob" diff --git a/test/contrib/rails/apps/application.rb b/test/contrib/rails/apps/application.rb index 028c47db71f..f2800732ebc 100644 --- a/test/contrib/rails/apps/application.rb +++ b/test/contrib/rails/apps/application.rb @@ -45,7 +45,6 @@ def test_config c.use :rails c.use :redis end - Rails.application.config.active_job.queue_adapter = :sidekiq # Initialize the Rails application require 'contrib/rails/apps/controllers' diff --git a/test/contrib/rails/test_helper.rb b/test/contrib/rails/test_helper.rb index 26e9ff95e52..5ab9c02fa3c 100644 --- a/test/contrib/rails/test_helper.rb +++ b/test/contrib/rails/test_helper.rb @@ -68,7 +68,7 @@ def database_configuration logger.info "Testing against Rails #{Rails.version} with connector '#{connector}'" case Rails.version -when '5.0.1' +when '5.0.1', '5.1.5' require 'contrib/rails/apps/rails5' when '4.2.7.1' require 'contrib/rails/apps/rails4' From 1c6426cb5e131818c7fc8dfdd6a93c12052c5a96 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 30 Mar 2018 16:13:13 -0400 Subject: [PATCH 44/72] Changed: README install instructions for links to setup documentation. --- README.md | 121 +++++------------------------------------------------- 1 file changed, 10 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index b469d8b9e94..71b820bf704 100644 --- a/README.md +++ b/README.md @@ -1,123 +1,22 @@ -# dd-trace-rb +# Datadog Trace Client [![CircleCI](https://circleci.com/gh/DataDog/dd-trace-rb/tree/master.svg?style=svg&circle-token=b0bd5ef866ec7f7b018f48731bb495f2d1372cc1)](https://circleci.com/gh/DataDog/dd-trace-rb/tree/master) -## Documentation - -You can find the latest documentation on [rubydoc.info][docs] - -[docs]: http://gems.datadoghq.com/trace/docs/ +``ddtrace`` is Datadog’s tracing client for Ruby. It is used to trace requests as they flow across web servers, +databases and microservices so that developers have great visiblity into bottlenecks and troublesome requests. ## Getting started -### Install - -Install the Ruby client with the ``gem`` command: - -``` -gem install ddtrace -``` - -If you're using ``Bundler``, just update your ``Gemfile`` as follows: - -```ruby -source 'https://rubygems.org' - -# tracing gem -gem 'ddtrace' -``` - -To use a development/preview version, use: - -```ruby -gem 'ddtrace', :github => 'DataDog/dd-trace-rb', :branch => 'me/my-feature-branch' -``` - -### Quickstart (manual instrumentation) - -If you aren't using a supported framework instrumentation, you may want to to manually instrument your code. -Adding tracing to your code is very simple. As an example, let’s imagine we have a web server and we want -to trace requests to the home page: - -```ruby -require 'ddtrace' -require 'sinatra' -require 'active_record' - -# a generic tracer that you can use across your application -tracer = Datadog.tracer - -get '/' do - tracer.trace('web.request') do |span| - # set some span metadata - span.service = 'my-web-site' - span.resource = '/' - span.set_tag('http.method', request.request_method) - - # trace the activerecord call - tracer.trace('posts.fetch') do - @posts = Posts.order(created_at: :desc).limit(10) - end - - # trace the template rendering - tracer.trace('template.render') do - erb :index - end - end -end -``` - -### Quickstart (integration) - -Instead of doing the above manually, whenever an integration is available, -you can activate it. The example above would become: - -```ruby -require 'ddtrace' -require 'sinatra' -require 'active_record' - -Datadog.configure do |c| - c.use :sinatra - c.use :active_record -end - -# now write your code naturally, it's traced automatically -get '/' do - @posts = Posts.order(created_at: :desc).limit(10) - erb :index -end -``` - -This will automatically trace any app inherited from `Sinatra::Application`. -To trace apps inherited from `Sinatra::Base`, you should manually register -the tracer inside your class. - -```ruby -require "ddtrace" -require "ddtrace/contrib/sinatra/tracer" - -class App < Sinatra::Base - register Datadog::Contrib::Sinatra::Tracer -end -``` - -To configure the Datadog Tracer, you can define the `configure` block as follows: - -```ruby -Datadog.configure do |c| - c.tracer enabled: false, hostname: 'trace-agent.local' - # [...] -end -``` - -For a list of available options, check the [Tracer documentation](http://gems.datadoghq.com/trace/docs/#Configure_the_tracer). +For installation instructions, check out our [setup documenation][setup docs]. +For configuration instructions and details about using the API, check out our [API documentation][api docs] and [gem documentation][gem docs]. -To know if a given framework or lib is supported by our client, -please consult our [integrations][contrib] list. +For descriptions of terminology used in APM, take a look at the [official documentation][terminology docs]. -[contrib]: http://www.rubydoc.info/github/DataDog/dd-trace-rb/Datadog/Contrib +[setup docs]: https://docs.datadoghq.com/tracing/setup/ruby/ +[api docs]: https://github.com/DataDog/dd-trace-rb/blob/master/docs/GettingStarted.md +[gem docs]: http://gems.datadoghq.com/trace/docs/ +[terminology docs]: https://docs.datadoghq.com/tracing/terminology/ ## Development From c65431b753b9c964ee1a96dd2774bd7dd1b8b5a9 Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 3 Apr 2018 15:50:11 -0400 Subject: [PATCH 45/72] Fixed: ActiveSupport::Notification subscriptions having issues with #ensure_clean_context! --- .../notifications/subscription.rb | 26 ++++++-- lib/ddtrace/contrib/racecar/patcher.rb | 22 ++++++- .../notifications/subscription_spec.rb | 60 +++++++++++++++++++ 3 files changed, 100 insertions(+), 8 deletions(-) diff --git a/lib/ddtrace/contrib/active_support/notifications/subscription.rb b/lib/ddtrace/contrib/active_support/notifications/subscription.rb index d460413126f..fde3891476d 100644 --- a/lib/ddtrace/contrib/active_support/notifications/subscription.rb +++ b/lib/ddtrace/contrib/active_support/notifications/subscription.rb @@ -16,10 +16,20 @@ def initialize(tracer, span_name, options, &block) @span_name = span_name @options = options @block = block + @before_trace_callbacks = [] + @after_trace_callbacks = [] + end + + def before_trace(&block) + @before_trace_callbacks << block if block_given? + end + + def after_trace(&block) + @after_trace_callbacks << block if block_given? end def start(_name, _id, _payload) - ensure_clean_context! + run_callbacks(@before_trace_callbacks) tracer.trace(@span_name, @options) end @@ -28,6 +38,7 @@ def finish(name, id, payload) return nil if span.nil? block.call(span, name, id, payload) span.finish + run_callbacks(@after_trace_callbacks) end end @@ -57,11 +68,14 @@ def subscribers @subscribers ||= {} end - private - - def ensure_clean_context! - return unless tracer.call_context.current_span - tracer.provider.context = Context.new + def run_callbacks(callbacks) + callbacks.each do |callback| + begin + callback.call + rescue StandardError => e + Datadog::Tracer.log.debug("ActiveSupport::Notifications callback failed: #{e.message}") + end + end end end end diff --git a/lib/ddtrace/contrib/racecar/patcher.rb b/lib/ddtrace/contrib/racecar/patcher.rb index a26d128501b..35e765a12b7 100644 --- a/lib/ddtrace/contrib/racecar/patcher.rb +++ b/lib/ddtrace/contrib/racecar/patcher.rb @@ -16,8 +16,17 @@ module Patcher option :service_name, default: 'racecar' on_subscribe do - subscribe('process_message.racecar', self::NAME_MESSAGE, {}, configuration[:tracer], &method(:process)) - subscribe('process_batch.racecar', self::NAME_BATCH, {}, configuration[:tracer], &method(:process)) + # Subscribe to single messages + subscription(self::NAME_MESSAGE, {}, configuration[:tracer], &method(:process)).tap do |subscription| + subscription.before_trace(&method(:ensure_clean_context!)) + subscription.subscribe('process_message.racecar') + end + + # Subscribe to batch messages + subscription(self::NAME_BATCH, {}, configuration[:tracer], &method(:process)).tap do |subscription| + subscription.before_trace(&method(:ensure_clean_context!)) + subscription.subscribe('process_batch.racecar') + end end class << self @@ -61,6 +70,15 @@ def configuration def compatible? defined?(::Racecar) && defined?(::ActiveSupport::Notifications) end + + # Context objects are thread-bound. + # If Racecar re-uses threads, context from a previous trace + # could leak into the new trace. This "cleans" current context, + # preventing such a leak. + def ensure_clean_context! + return unless tracer.call_context.current_span + tracer.provider.context = Context.new + end end end end diff --git a/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb b/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb index 00683239e43..d6cc97ff086 100644 --- a/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb +++ b/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb @@ -48,6 +48,66 @@ end end + describe '#before_trace' do + context 'given a block' do + let(:callback_block) { Proc.new { callback_spy.call } } + let(:callback_spy) { double('callback spy') } + before(:each) { subscription.before_trace(&callback_block) } + + shared_examples_for 'a before_trace callback' do + context 'on #start' do + it do + expect(callback_spy).to receive(:call).ordered + expect(tracer).to receive(:trace).ordered + subscription.start(double('name'), double('id'), double('payload')) + end + end + end + + context 'that doesn\'t raise an error' do + let(:callback_block) { Proc.new { callback_spy.call } } + it_behaves_like 'a before_trace callback' + end + + context 'that raises an error' do + let(:callback_block) { Proc.new { callback_spy.call; raise ArgumentError.new('Fail!') } } + it_behaves_like 'a before_trace callback' + end + end + end + + describe '#after_trace' do + context 'given a block' do + let(:callback_block) { Proc.new { callback_spy.call } } + let(:callback_spy) { double('callback spy') } + before(:each) { subscription.after_trace(&callback_block) } + + shared_examples_for 'an after_trace callback' do + context 'on #finish' do + let(:span) { instance_double(Datadog::Span) } + + it do + expect(tracer).to receive(:active_span).and_return(span).ordered + expect(spy).to receive(:call).ordered + expect(span).to receive(:finish).ordered + expect(callback_spy).to receive(:call).ordered + subscription.finish(double('name'), double('id'), double('payload')) + end + end + end + + context 'that doesn\'t raise an error' do + let(:callback_block) { Proc.new { callback_spy.call } } + it_behaves_like 'an after_trace callback' + end + + context 'that raises an error' do + let(:callback_block) { Proc.new { callback_spy.call; raise ArgumentError.new('Fail!') } } + it_behaves_like 'an after_trace callback' + end + end + end + describe '#subscribe' do subject(:result) { subscription.subscribe(pattern) } let(:pattern) { double('pattern') } From b3ab5f5ece1dd2c94526083520beb98fd2102b76 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 30 Mar 2018 16:13:35 -0400 Subject: [PATCH 46/72] Changed: GettingStarted.md formatting. --- README.md | 26 +- docs/GettingStarted.md | 1178 ++++++++++++++++++++++++---------------- 2 files changed, 700 insertions(+), 504 deletions(-) diff --git a/README.md b/README.md index 71b820bf704..40539fbb272 100644 --- a/README.md +++ b/README.md @@ -7,16 +7,16 @@ databases and microservices so that developers have great visiblity into bottlen ## Getting started -For installation instructions, check out our [setup documenation][setup docs]. +For a basic product overview, check out our [setup documentation][setup docs]. -For configuration instructions and details about using the API, check out our [API documentation][api docs] and [gem documentation][gem docs]. +For installation, configuration, and details about using the API, check out our [API documentation][api docs] and [gem documentation][gem docs]. -For descriptions of terminology used in APM, take a look at the [official documentation][terminology docs]. +For descriptions of terminology used in APM, take a look at the [official documentation][visualization docs]. [setup docs]: https://docs.datadoghq.com/tracing/setup/ruby/ [api docs]: https://github.com/DataDog/dd-trace-rb/blob/master/docs/GettingStarted.md [gem docs]: http://gems.datadoghq.com/trace/docs/ -[terminology docs]: https://docs.datadoghq.com/tracing/terminology/ +[visualization docs]: https://docs.datadoghq.com/tracing/visualization/ ## Development @@ -35,23 +35,7 @@ You can launch tests using the following Rake commands: ... Run ``rake --tasks`` for the list of available Rake tasks. - -Available appraisals are: - -* ``contrib``: default for integrations -* ``contrib-old``: default for integrations, with version suited for old Ruby (possibly unmaintained) versions -* ``rails3-mysql2``: Rails3 with Mysql -* ``rails3-postgres``: Rails 3 with Postgres -* ``rails3-postgres-redis``: Rails 3 with Postgres and Redis -* ``rails3-postgres-sidekiq``: Rails 3 with Postgres and Sidekiq -* ``rails4-mysql2``: Rails4 with Mysql -* ``rails4-postgres``: Rails 4 with Postgres -* ``rails4-postgres-redis``: Rails 4 with Postgres and Redis -* ``rails4-postgres-sidekiq``: Rails 4 with Postgres and Sidekiq -* ``rails5-mysql2``: Rails5 with Mysql -* ``rails5-postgres``: Rails 5 with Postgres -* ``rails5-postgres-redis``: Rails 5 with Postgres and Redis -* ``rails5-postgres-sidekiq``: Rails 5 with Postgres and Sidekiq +Run ``appraisal list`` for the list of available appraisals. The test suite requires many backing services (PostgreSQL, MySQL, Redis, ...) and we're using ``docker`` and ``docker-compose`` to start these services in the CI. diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index 557f11bc48a..a997c256f41 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -3,257 +3,407 @@ ``ddtrace`` is Datadog’s tracing client for Ruby. It is used to trace requests as they flow across web servers, databases and microservices so that developers have great visiblity into bottlenecks and troublesome requests. -## Install the gem - -Install the tracing client, adding the following gem in your ``Gemfile``: - +## Getting started + +For a basic product overview, check out our [setup documentation][setup docs]. + +For details about contributing, check out the [development guide][development docs]. + +For descriptions of terminology used in APM, take a look at the [official documentation][visualization docs]. + +[setup docs]: https://docs.datadoghq.com/tracing/setup/ruby/ +[development docs]: https://github.com/DataDog/dd-trace-rb/blob/master/README.md#development +[visualization docs]: https://docs.datadoghq.com/tracing/visualization/ + +## Table of Contents + + - [Compatibility](#compatibility) + - [Installation](#installation) + - [Quickstart for Rails applications](#quickstart-for-rails-applications) + - [Quickstart for Ruby applications](#quickstart-for-ruby-applications) + - [Manual instrumentation](#manual-instrumentation) + - [Integration instrumentation](#integration-instrumentation) + - [Active Record](#active-record) + - [AWS](#aws) + - [Dalli](#dalli) + - [Elastic Search](#elastic-search) + - [Faraday](#faraday) + - [Grape](#grape) + - [GraphQL](#graphql) + - [MongoDB](#mongodb) + - [Net/HTTP](#nethttp) + - [Racecar](#racecar) + - [Rack](#rack) + - [Rails](#rails) + - [Redis](#redis) + - [Resque](#resque) + - [Sidekiq](#sidekiq) + - [Sinatra](#sinatra) + - [Sucker Punch](#sucker-punch) + - [Advanced configuration](#advanced-configuration) + - [Tracer settings](#tracer-settings) + - [Custom logging](#custom-logging) + - [Environment and tags](#environment-and-tags) + - [Sampling](#sampling) + - [Priority sampling](#priority-sampling) + - [Distributed tracing](#distributed-tracing) + - [Processing pipeline](#processing-pipeline) + - [Filtering](#filtering) + - [Processing](#processing) + +## Compatibility + +**Supported Ruby interpreters**: + +| Type | Documentation | Version | Support type | +| ----- | -------------------------- | ----- | ------------ | +| MRI | https://www.ruby-lang.org/ | 1.9.1 | Experimental | +| | | 1.9.3 | Full | +| | | 2.0 | Full | +| | | 2.1 | Full | +| | | 2.2 | Full | +| | | 2.3 | Full | +| | | 2.4 | Full | +| JRuby | http://jruby.org/ | 9.1.5 | Experimental | + +*Full* support indicates all tracer features are available. + +*Experimental* indicates most features should be available, but unverified. + +**Supported web servers**: + +| Type | Documentation | Version | Support type | +| --------- | --------------------------------- | ------------ | ------------ | +| Puma | http://puma.io/ | 2.16+ / 3.6+ | Full | +| Unicorn | https://bogomips.org/unicorn/ | 4.8+ / 5.1+ | Full | +| Passenger | https://www.phusionpassenger.com/ | 5.0+ | Full | + +## Installation + +The following steps will help you quickly start tracing your Ruby application. + +### Setup the Datadog Agent + +The Ruby APM tracer sends trace data through the Datadog Agent. + +[Install and configure the Datadog Agent](https://docs.datadoghq.com/tracing/setup), see additional documentation for [tracing Docker applications](https://docs.datadoghq.com/tracing/setup/docker/). + +### Quickstart for Rails applications + +1. Add the `ddtrace` gem to your Gemfile: + + ```ruby source 'https://rubygems.org' - - # tracing gem gem 'ddtrace' + ``` -If you're not using ``Bundler`` to manage your dependencies, you can install ``ddtrace`` with: - - gem install ddtrace +2. Install the gem with `bundle install` +3. Create a `config/initializers/datadog.rb` file containing: -We strongly suggest pinning the version of the library you deploy. + ```ruby + Datadog.configure do |c| + # This will activate auto-instrumentation for Rails + c.use :rails + end + ``` -## Quickstart + You can also activate additional integrations here (see [Integration instrumentation](#integration-instrumentation)) -The easiest way to get started with the tracing client is to instrument your web application. -All configuration is done through ``Datadog.configure`` method. As an -example, below is a setup that enables auto instrumentation for Rails, Redis and -Grape, and sets a custom endpoint for the trace agent: +### Quickstart for Ruby applications - # config/initializers/datadog-tracer.rb +1. Install the gem with `gem install ddtrace` +2. Add a configuration block to your Ruby application: + ```ruby + require 'ddtrace' Datadog.configure do |c| - c.tracer hostname: 'trace-agent.local' - c.use :rails - c.use :grape - c.use :redis, service_name: 'cache' + # Configure the tracer here. + # Activate integrations, change tracer settings, etc... + # By default without additional configuration, nothing will be traced. end + ``` -For further details and options, check our integrations list. +3. Add or activate instrumentation by doing either of the following: + 1. Activate integration instrumentation (see [Integration instrumentation](#integration-instrumentation)) + 2. Add manual instrumentation around your code (see [Manual instrumentation](#manual-instrumentation)) -## Available Integrations +### Final steps for installation -* [Ruby on Rails](#Ruby_on_Rails) -* [Sinatra](#Sinatra) -* [Rack](#Rack) -* [GraphQL](#GraphQL) -* [Grape](#Grape) -* [Active Record](#Active_Record) -* [Elastic Search](#Elastic_Search) -* [MongoDB](#MongoDB) -* [Sidekiq](#Sidekiq) -* [Resque](#Resque) -* [SuckerPunch](#SuckerPunch) -* [Net/HTTP](#Net_HTTP) -* [Faraday](#Faraday) -* [Dalli](#Dalli) -* [Redis](#Redis) +After setting up, your services will appear on the [APM services page](https://app.datadoghq.com/apm/services) within a few minutes. Learn more about [using the APM UI][visualization docs]. -## Web Frameworks +## Manual Instrumentation -### Ruby on Rails +If you aren't using a supported framework instrumentation, you may want to to manually instrument your code. -The Rails integration will trace requests, database calls, templates rendering and cache read/write/delete -operations. The integration makes use of the Active Support Instrumentation, listening to the Notification API -so that any operation instrumented by the API is traced. +To trace any Ruby code, you can use the `Datadog.tracer.trace` method: -To enable the Rails auto instrumentation, create an initializer file in your ``config/`` folder: +```ruby +Datadog.tracer.trace(name, options) do |span| + # Wrap this block around the code you want to instrument + # Additionally, you can modify the span here. + # e.g. Change the resource name, set tags, etc... +end +``` - # config/initializers/datadog-tracer.rb +Where `name` should be a `String` that describes the generic kind of operation being done (e.g. `'web.request'`, or `'request.parse'`) - Datadog.configure do |c| - c.use :rails, options +And `options` is an optional `Hash` that accepts the following parameters: + +| Key | Type | Description | Default | +| --- | --- | --- | --- | +| ``service`` | `String` | The service name which this span belongs (e.g. `'my-web-service'`) | Tracer `default-service`, `$PROGRAM_NAME` or `'ruby'` | +| ``resource`` | `String` | Name of the resource or action being operated on. Traces with the same resource value will be grouped together for the purpose of metrics (but still independently viewable.) Usually domain specific, such as a URL, query, request, etc. (e.g. `'Article#submit'`, `http://example.com/articles/list`.) | `name` of Span. | +| ``span_type`` | `String` | The type of the span (such as `'http'`, `'db'`, etc.) | `nil` | +| ``child_of`` | `Datadog::Span` / `Datadog::Context` | Parent for this span. If not provided, will automatically become current active span. | `nil` | +| ``start_time`` | `Integer` | When the span actually starts. Useful when tracing events that have already happened. | `Time.now.utc` | +| ``tags`` | `Hash` | Extra tags which should be added to the span. | `{}` | + +It's highly recommended you set both `service` and `resource` at a minimum. Spans without a `service` or `resource` as `nil` will be discarded by the Datadog agent. + +Example of manual instrumentation in action: + +```ruby +get '/posts' do + Datadog.tracer.trace('web.request', service: 'my-blog', resource: 'GET /posts') do |span| + # Trace the activerecord call + Datadog.tracer.trace('posts.fetch') do + @posts = Posts.order(created_at: :desc).limit(10) end -Where `options` is an optional `Hash` that accepts the following parameters: + # Add some APM tags + span.set_tag('http.method', request.request_method) + span.set_tag('posts.count', @posts.length) + # Trace the template rendering + Datadog.tracer.trace('template.render') do + erb :index + end + end +end +``` -| Key | Description | Default | -| --- | --- | --- | -| ``service_name`` | Service name used when tracing application requests (on the `rack` level) | ```` (inferred from your Rails application namespace) | -| ``controller_service`` | Service name used when tracing a Rails action controller | ``-controller`` | -| ``cache_service`` | Cache service name used when tracing cache activity | ``-cache`` | -| ``database_service`` | Database service name used when tracing database activity | ``-`` | -| ``exception_controller`` | Class or Module which identifies a custom exception controller class. Tracer provides improved error behavior when it can identify custom exception controllers. By default, without this option, it 'guesses' what a custom exception controller looks like. Providing this option aids this identification. | ``nil`` | -| ``distributed_tracing`` | Enables [distributed tracing](#Distributed_Tracing) so that this service trace is connected with a trace of another service if tracing headers are received | `false` | -| ``middleware_names`` | Enables any short-circuited middleware requests to display the middleware name as resource for the trace. | `false` | -| ``template_base_path`` | Used when the template name is parsed. If you don't store your templates in the ``views/`` folder, you may need to change this value | ``views/`` | -| ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | +**Asynchronous tracing** -### Sinatra +It might not always be possible to wrap `Datadog.tracer.trace` around a block of code. Some event or notification based instrumentation might only notify you when an event begins or ends. -The Sinatra integration traces requests and template rendering. +To trace these operations, you can trace code asynchronously by calling `Datadog.tracer.trace` without a block: -To start using the tracing client, make sure you import ``ddtrace`` and ``ddtrace/contrib/sinatra/tracer`` after -either ``sinatra`` or ``sinatra/base``: +```ruby +# Some instrumentation framework calls this after an event began and finished... +def db_query(start, finish, query) + span = Datadog.tracer.trace('database.query') + span.resource = query + span.start_time = start + span.finish(finish) +end +``` - require 'sinatra' - require 'ddtrace' - require 'ddtrace/contrib/sinatra/tracer' +Calling `Datadog.tracer.trace` without a block will cause the function to return a `Datadog::Span` that is started, but not finished. You can then modify this span however you wish, then close it `finish`. - Datadog.configure do |c| - c.use :sinatra, options - end +*You must not leave any unfinished spans.* If any spans are left open when the trace completes, the trace will be discarded. You can [activate debug mode](#tracer-settings) to check for warnings if you suspect this might be happening. - get '/' do - 'Hello world!' - end +To avoid this scenario when handling start/finish events, you can use `Datadog.tracer.active_span` to get the current active span. -Where `options` is an optional `Hash` that accepts the following parameters: +```ruby +# e.g. ActiveSupport::Notifications calls this when an event starts +def start(name, id, payload) + # Start a span + Datadog.tracer.trace(name) +end -| Key | Description | Default | -| --- | --- | --- | -| ``service_name`` | Service name used for `sinatra` instrumentation | sinatra | -| ``resource_script_names`` | Prepend resource names with script name | ``false`` | -| ``distributed_tracing`` | Enables [distributed tracing](#Distributed_Tracing) so that this service trace is connected with a trace of another service if tracing headers are received | `false` | -| ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | +# e.g. ActiveSupport::Notifications calls this when an event finishes +def finish(name, id, payload) + # Retrieve current active span (thread-safe) + current_span = Datadog.tracer.active_span + unless current_span.nil? + current_span.resource = payload[:query] + current_span.finish + end +end +``` -### Rack +## Integration instrumentation -The Rack integration provides a middleware that traces all requests before they reach the underlying framework -or application. It responds to the Rack minimal interface, providing reasonable values that can be -retrieved at the Rack level. This integration is automatically activated with web frameworks like Rails. -If you're using a plain Rack application, just enable the integration it to your ``config.ru``: +Many popular libraries and frameworks are supported out-of-the-box, which can be auto-instrumented. Although they are not activated automatically, they can be easily activated and configured by using the `Datadog.configure` API: - # config.ru example - require 'ddtrace' +```ruby +Datadog.configure do |c| + # Activates and configures an integration + c.use :integration_name, options +end +``` - Datadog.configure do |c| - c.use :rack, options - end +`options` is a `Hash` of integration-specific configuration settings. + +For a list of available integrations, and their configuration options, please refer to the following: + +| Name | Key | Versions Supported | How to configure | Gem source | +| -------------- | --------------- | ---------------------- | ------------------------- | ------------------------------------------------------------------------------ | +| Active Record | `active_record` | `>= 3.2, < 5.2` | *[Link](#active-record)* | *[Link](https://github.com/rails/rails/tree/master/activerecord)* | +| AWS | `aws` | `>= 2.0` | *[Link](#aws)* | *[Link](https://github.com/aws/aws-sdk-ruby)* | +| Dalli | `dalli` | `>= 2.7` | *[Link](#dalli)* | *[Link](https://github.com/petergoldstein/dalli)* | +| Elastic Search | `elasticsearch` | `>= 6.0` | *[Link](#elastic-search)* | *[Link](https://github.com/elastic/elasticsearch-ruby)* | +| Faraday | `faraday` | `>= 0.14` | *[Link](#faraday)* | *[Link](https://github.com/lostisland/faraday)* | +| Grape | `grape` | `>= 1.0` | *[Link](#grape)* | *[Link](https://github.com/ruby-grape/grape)* | +| GraphQL | `graphql` | `>= 1.7.9` | *[Link](#graphql)* | *[Link](https://github.com/rmosolgo/graphql-ruby)* | +| MongoDB | `mongo` | `>= 2.0, < 2.5` | *[Link](#mongodb)* | *[Link](https://github.com/mongodb/mongo-ruby-driver)* | +| Net/HTTP | `http` | *(Any supported Ruby)* | *[Link](#nethttp)* | *[Link](https://ruby-doc.org/stdlib-2.4.0/libdoc/net/http/rdoc/Net/HTTP.html)* | +| Racecar | `racecar` | `>= 0.3.5` | *[Link](#racecar)* | *[Link](https://github.com/zendesk/racecar)* | +| Rack | `rack` | `>= 1.4.7` | *[Link](#rack)* | *[Link](https://github.com/rack/rack)* | +| Rails | `rails` | `>= 3.2, < 5.2` | *[Link](#rails)* | *[Link](https://github.com/rails/rails)* | +| Redis | `redis` | `>= 3.2, < 4.0` | *[Link](#redis)* | *[Link](https://github.com/redis/redis-rb)* | +| Resque | `resque` | `>= 1.0, < 2.0` | *[Link](#resque)* | *[Link](https://github.com/resque/resque)* | +| Sidekiq | `sidekiq` | `>= 4.0` | *[Link](#sidekiq)* | *[Link](https://github.com/mperham/sidekiq)* | +| Sinatra | `sinatra` | `>= 1.4.5` | *[Link](#sinatra)* | *[Link](https://github.com/sinatra/sinatra)* | +| Sucker Punch | `sucker_punch` | `>= 2.0` | *[Link](#sucker-punch)* | *[Link](https://github.com/brandonhilkert/sucker_punch)* | - use Datadog::Contrib::Rack::TraceMiddleware +### Active Record - app = proc do |env| - [ 200, {'Content-Type' => 'text/plain'}, ['OK'] ] - end +Most of the time, Active Record is set up as part of a web framework (Rails, Sinatra...) however it can be set up alone: - run app +```ruby +require 'tmpdir' +require 'sqlite3' +require 'active_record' +require 'ddtrace' + +Datadog.configure do |c| + c.use :active_record, options +end + +Dir::Tmpname.create(['test', '.sqlite']) do |db| + conn = ActiveRecord::Base.establish_connection(adapter: 'sqlite3', + database: db) + conn.connection.execute('SELECT 42') # traced! +end +``` Where `options` is an optional `Hash` that accepts the following parameters: | Key | Description | Default | | --- | --- | --- | -| ``service_name`` | Service name used when tracing application requests | rack | -| ``distributed_tracing`` | Enables [distributed tracing](#Distributed_Tracing) so that this service trace is connected with a trace of another service if tracing headers are received | `false` | -| ``middleware_names`` | Enable this if you want to use the middleware classes as the resource names for `rack` spans. Must provide the ``application`` option with it. | ``false`` | -| ``quantize`` | Hash containing options for quantization. May include `:query` or `:fragment`. | {} | -| ``quantize.query`` | Hash containing options for query portion of URL quantization. May include `:show` or `:exclude`. See options below. Option must be nested inside the `quantize` option. | {} | -| ``quantize.query.show`` | Defines which values should always be shown. Shows no values by default. May be an Array of strings, or `:all` to show all values. Option must be nested inside the `query` option. | ``nil`` | -| ``quantize.query.exclude`` | Defines which values should be removed entirely. Excludes nothing by default. May be an Array of strings, or `:all` to remove the query string entirely. Option must be nested inside the `query` option. | ``nil`` | -| ``quantize.fragment`` | Defines behavior for URL fragments. Removes fragments by default. May be `:show` to show URL fragments. Option must be nested inside the `quantize` option. | ``nil`` | -| ``application`` | Your Rack application. Necessary for enabling middleware resource names. | ``nil`` | -| ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | +| ``service_name`` | Service name used for database portion of `active_record` instrumentation. | Name of database adapter (e.g. `mysql2`) | +| ``orm_service_name`` | Service name used for the Ruby ORM portion of `active_record` instrumentation. Overrides service name for ORM spans if explicitly set, which otherwise inherit their service from their parent. | ``active_record`` | -Configuring URL quantization behavior: +### AWS + +The AWS integration will trace every interaction (e.g. API calls) with AWS services (S3, ElastiCache etc.). + +```ruby +require 'aws-sdk' +require 'ddtrace' -``` Datadog.configure do |c| - # Default behavior: all values are quantized, fragment is removed. - # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id&sort_by - # http://example.com/path?categories[]=1&categories[]=2 --> http://example.com/path?categories[] + c.use :aws, options +end - # Show values for any query string parameter matching 'category_id' exactly - # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id=1&sort_by - c.use :rack, quantize: { query: { show: ['category_id'] } } +Aws::S3::Client.new.list_buckets # traced call +``` - # Show all values for all query string parameters - # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id=1&sort_by=asc - c.use :rack, quantize: { query: { show: :all } } +Where `options` is an optional `Hash` that accepts the following parameters: - # Totally exclude any query string parameter matching 'sort_by' exactly - # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id - c.use :rack, quantize: { query: { exclude: ['sort_by'] } } +| Key | Description | Default | +| --- | --- | --- | +| ``service_name`` | Service name used for `aws` instrumentation | aws | - # Remove the query string entirely - # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path - c.use :rack, quantize: { query: { exclude: :all } } +### Dalli - # Show URL fragments - # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id&sort_by#featured - c.use :rack, quantize: { fragment: :show } +Dalli integration will trace all calls to your ``memcached`` server: + +```ruby +require 'dalli' +require 'ddtrace' + +Datadog.configure do |c| + c.use :dalli, service_name: 'dalli' end + +client = Dalli::Client.new('localhost:11211', options) +client.set('abc', 123) ``` -## Other libraries +Where `options` is an optional `Hash` that accepts the following parameters: -### GraphQL +| Key | Description | Default | +| --- | --- | --- | +| ``service_name`` | Service name used for `dalli` instrumentation | memcached | -*Version 1.7.9+ supported* +### Elastic Search -The GraphQL integration activates instrumentation for GraphQL queries. To activate your integration, use the ``Datadog.configure`` method: +The Elasticsearch integration will trace any call to ``perform_request`` in the ``Client`` object: - # Inside Rails initializer or equivalent - Datadog.configure do |c| - c.use :graphql, - service_name: 'graphql', - schemas: [YourSchema] - end +```ruby +require 'elasticsearch/transport' +require 'ddtrace' - # Then run a GraphQL query - YourSchema.execute(query, variables: {}, context: {}, operation_name: nil) +Datadog.configure do |c| + c.use :elasticsearch, options +end -The `use :graphql` method accepts the following parameters: +# now do your Elastic Search stuff, eg: +client = Elasticsearch::Client.new url: 'http://127.0.0.1:9200' +response = client.perform_request 'GET', '_cluster/health' +``` + +Where `options` is an optional `Hash` that accepts the following parameters: | Key | Description | Default | | --- | --- | --- | -| ``service_name`` | Service name used for `graphql` instrumentation | ``ruby-graphql`` | -| ``schemas`` | Required. Array of `GraphQL::Schema` objects which to trace. Tracing will be added to all the schemas listed, using the options provided to this configuration. If you do not provide any, then tracing will not be activated. | ``[]`` | -| ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | +| ``service_name`` | Service name used for `elasticsearch` instrumentation | elasticsearch | +| ``quantize`` | Hash containing options for quantization. May include `:show` with an Array of keys to not quantize (or `:all` to skip quantization), or `:exclude` with Array of keys to exclude entirely. | {} | + +### Faraday -##### Manually configuring GraphQL schemas +The `faraday` integration is available through the `ddtrace` middleware: -If you prefer to individually configure the tracer settings for a schema (e.g. you have multiple schemas with different service names), -in the schema definition, you can add the following [using the GraphQL API](http://graphql-ruby.org/queries/tracing.html): +```ruby +require 'faraday' +require 'ddtrace' -``` -YourSchema = GraphQL::Schema.define do - use( - GraphQL::Tracing::DataDogTracing, - service: 'graphql' - ) +Datadog.configure do |c| + c.use :faraday, service_name: 'faraday' # global service name end -``` -Or you can modify an already defined schema: - -``` -YourSchema.define do - use( - GraphQL::Tracing::DataDogTracing, - service: 'graphql' - ) +connection = Faraday.new('https://example.com') do |builder| + builder.use(:ddtrace, options) + builder.adapter Faraday.default_adapter end + +connection.get('/foo') ``` -Do *not* `use :graphql` in `Datadog.configure` if you choose to configure manually, as to avoid double tracing. These two means of configuring GraphQL tracing are considered mutually exclusive. +Where `options` is an optional `Hash` that accepts the following parameters: + +| Key | Default | Description | +| --- | --- | --- | +| `service_name` | Global service name (default: `faraday`) | Service name for this specific connection object. | +| `split_by_domain` | `false` | Uses the request domain as the service name when set to `true`. | +| `distributed_tracing` | `false` | Propagates tracing context along the HTTP request when set to `true`. | +| `error_handler` | ``5xx`` evaluated as errors | A callable object that receives a single argument – the request environment. If it evaluates to a *truthy* value, the trace span is marked as an error. | ### Grape -The Grape integration adds the instrumentation to Grape endpoints and filters. This integration can work side by side -with other integrations like Rack and Rails. To activate your integration, use the ``Datadog.configure`` method before -defining your Grape application: +The Grape integration adds the instrumentation to Grape endpoints and filters. This integration can work side by side with other integrations like Rack and Rails. - # api.rb - require 'grape' - require 'ddtrace' +To activate your integration, use the ``Datadog.configure`` method before defining your Grape application: - Datadog.configure do |c| - c.use :grape, options - end +```ruby +# api.rb +require 'grape' +require 'ddtrace' - # then define your application - class RackTestingAPI < Grape::API - desc 'main endpoint' - get :success do - 'Hello world!' - end - end +Datadog.configure do |c| + c.use :grape, options +end + +# then define your application +class RackTestingAPI < Grape::API + desc 'main endpoint' + get :success do + 'Hello world!' + end +end +``` Where `options` is an optional `Hash` that accepts the following parameters: @@ -261,77 +411,78 @@ Where `options` is an optional `Hash` that accepts the following parameters: | --- | --- | --- | | ``service_name`` | Service name used for `grape` instrumentation | grape | -### Active Record +### GraphQL -Most of the time, Active Record is set up as part of a web framework (Rails, Sinatra...) -however it can be set up alone: +The GraphQL integration activates instrumentation for GraphQL queries. - require 'tmpdir' - require 'sqlite3' - require 'active_record' - require 'ddtrace' +To activate your integration, use the ``Datadog.configure`` method: - Datadog.configure do |c| - c.use :active_record, options - end +```ruby +# Inside Rails initializer or equivalent +Datadog.configure do |c| + c.use :graphql, + service_name: 'graphql', + schemas: [YourSchema] +end - Dir::Tmpname.create(['test', '.sqlite']) do |db| - conn = ActiveRecord::Base.establish_connection(adapter: 'sqlite3', - database: db) - conn.connection.execute('SELECT 42') # traced! - end +# Then run a GraphQL query +YourSchema.execute(query, variables: {}, context: {}, operation_name: nil) +``` -Where `options` is an optional `Hash` that accepts the following parameters: +The `use :graphql` method accepts the following parameters: | Key | Description | Default | | --- | --- | --- | -| ``service_name`` | Service name used for database portion of `active_record` instrumentation. | Name of database adapter (e.g. `mysql2`) | -| ``orm_service_name`` | Service name used for the Ruby ORM portion of `active_record` instrumentation. Overrides service name for ORM spans if explicitly set, which otherwise inherit their service from their parent. | ``active_record`` | - -### Elastic Search +| ``service_name`` | Service name used for `graphql` instrumentation | ``ruby-graphql`` | +| ``schemas`` | Required. Array of `GraphQL::Schema` objects which to trace. Tracing will be added to all the schemas listed, using the options provided to this configuration. If you do not provide any, then tracing will not be activated. | ``[]`` | +| ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | -The Elasticsearch integration will trace any call to ``perform_request`` -in the ``Client`` object: +**Manually configuring GraphQL schemas** - require 'elasticsearch/transport' - require 'ddtrace' +If you prefer to individually configure the tracer settings for a schema (e.g. you have multiple schemas with different service names), in the schema definition, you can add the following [using the GraphQL API](http://graphql-ruby.org/queries/tracing.html): - Datadog.configure do |c| - c.use :elasticsearch, options - end +```ruby +YourSchema = GraphQL::Schema.define do + use( + GraphQL::Tracing::DataDogTracing, + service: 'graphql' + ) +end +``` - # now do your Elastic Search stuff, eg: - client = Elasticsearch::Client.new url: 'http://127.0.0.1:9200' - response = client.perform_request 'GET', '_cluster/health' +Or you can modify an already defined schema: -Where `options` is an optional `Hash` that accepts the following parameters: +```ruby +YourSchema.define do + use( + GraphQL::Tracing::DataDogTracing, + service: 'graphql' + ) +end +``` -| Key | Description | Default | -| --- | --- | --- | -| ``service_name`` | Service name used for `elasticsearch` instrumentation | elasticsearch | -| ``quantize`` | Hash containing options for quantization. May include `:show` with an Array of keys to not quantize (or `:all` to skip quantization), or `:exclude` with Array of keys to exclude entirely. | {} | +Do *not* `use :graphql` in `Datadog.configure` if you choose to configure manually, as to avoid double tracing. These two means of configuring GraphQL tracing are considered mutually exclusive. ### MongoDB -The integration traces any `Command` that is sent from the -[MongoDB Ruby Driver](https://github.com/mongodb/mongo-ruby-driver) to a MongoDB cluster. -By extension, Object Document Mappers (ODM) such as Mongoid are automatically instrumented -if they use the official Ruby driver. To activate the integration, simply: +The integration traces any `Command` that is sent from the [MongoDB Ruby Driver](https://github.com/mongodb/mongo-ruby-driver) to a MongoDB cluster. By extension, Object Document Mappers (ODM) such as Mongoid are automatically instrumented if they use the official Ruby driver. To activate the integration, simply: - require 'mongo' - require 'ddtrace' +```ruby +require 'mongo' +require 'ddtrace' - Datadog.configure do |c| - c.use :mongo, options - end +Datadog.configure do |c| + c.use :mongo, options +end - # now create a MongoDB client and use it as usual: - client = Mongo::Client.new([ '127.0.0.1:27017' ], :database => 'artists') - collection = client[:people] - collection.insert_one({ name: 'Steve' }) +# now create a MongoDB client and use it as usual: +client = Mongo::Client.new([ '127.0.0.1:27017' ], :database => 'artists') +collection = client[:people] +collection.insert_one({ name: 'Steve' }) - # In case you want to override the global configuration for a certain client instance - Datadog.configure(client, service_name: 'mongodb-primary') +# In case you want to override the global configuration for a certain client instance +Datadog.configure(client, service_name: 'mongodb-primary') +``` Where `options` is an optional `Hash` that accepts the following parameters: @@ -341,22 +492,23 @@ Where `options` is an optional `Hash` that accepts the following parameters: ### Net/HTTP -The Net/HTTP integration will trace any HTTP call using the standard lib -Net::HTTP module. +The Net/HTTP integration will trace any HTTP call using the standard lib Net::HTTP module. - require 'net/http' - require 'ddtrace' +```ruby +require 'net/http' +require 'ddtrace' - Datadog.configure do |c| - c.use :http, options - end +Datadog.configure do |c| + c.use :http, options +end - Net::HTTP.start('127.0.0.1', 8080) do |http| - request = Net::HTTP::Get.new '/index' - response = http.request request - end +Net::HTTP.start('127.0.0.1', 8080) do |http| + request = Net::HTTP::Get.new '/index' + response = http.request request +end - content = Net::HTTP.get(URI('http://127.0.0.1/index.html')) +content = Net::HTTP.get(URI('http://127.0.0.1/index.html')) +``` Where `options` is an optional `Hash` that accepts the following parameters: @@ -367,90 +519,144 @@ Where `options` is an optional `Hash` that accepts the following parameters: If you wish to configure each connection object individually, you may use the ``Datadog.configure`` as it follows: - client = Net::HTTP.new(host, port) - Datadog.configure(client, options) +```ruby +client = Net::HTTP.new(host, port) +Datadog.configure(client, options) +``` -### Faraday +### Racecar -The `faraday` integration is available through the `ddtrace` middleware: +The Racecar integration provides tracing for Racecar jobs. - require 'faraday' - require 'ddtrace' +You can enable it through `Datadog.configure`: - Datadog.configure do |c| - c.use :faraday, service_name: 'faraday' # global service name - end - - connection = Faraday.new('https://example.com') do |builder| - builder.use(:ddtrace, options) - builder.adapter Faraday.default_adapter - end +```ruby +require 'ddtrace' - connection.get('/foo') +Datadog.configure do |c| + c.use :racecar, options +end +``` Where `options` is an optional `Hash` that accepts the following parameters: -| Key | Default | Description | +| Key | Description | Default | | --- | --- | --- | -| `service_name` | Global service name (default: `faraday`) | Service name for this specific connection object. | -| `split_by_domain` | `false` | Uses the request domain as the service name when set to `true`. | -| `distributed_tracing` | `false` | Propagates tracing context along the HTTP request when set to `true`. | -| `error_handler` | ``5xx`` evaluated as errors | A callable object that receives a single argument – the request environment. If it evaluates to a *truthy* value, the trace span is marked as an error. | +| ``service_name`` | Service name used for `racecar` instrumentation | racecar | +| ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | -### AWS +### Rack -The AWS integration will trace every interaction (e.g. API calls) with AWS -services (S3, ElastiCache etc.). +The Rack integration provides a middleware that traces all requests before they reach the underlying framework or application. It responds to the Rack minimal interface, providing reasonable values that can be retrieved at the Rack level. - require 'aws-sdk' - require 'ddtrace' +This integration is automatically activated with web frameworks like Rails. If you're using a plain Rack application, just enable the integration it to your ``config.ru``: - Datadog.configure do |c| - c.use :aws, options - end +```ruby +# config.ru example +require 'ddtrace' - Aws::S3::Client.new.list_buckets # traced call +Datadog.configure do |c| + c.use :rack, options +end + +use Datadog::Contrib::Rack::TraceMiddleware + +app = proc do |env| + [ 200, {'Content-Type' => 'text/plain'}, ['OK'] ] +end + +run app +``` Where `options` is an optional `Hash` that accepts the following parameters: | Key | Description | Default | | --- | --- | --- | -| ``service_name`` | Service name used for `aws` instrumentation | aws | +| ``service_name`` | Service name used when tracing application requests | rack | +| ``distributed_tracing`` | Enables [distributed tracing](#distributed-tracing) so that this service trace is connected with a trace of another service if tracing headers are received | `false` | +| ``middleware_names`` | Enable this if you want to use the middleware classes as the resource names for `rack` spans. Must provide the ``application`` option with it. | ``false`` | +| ``quantize`` | Hash containing options for quantization. May include `:query` or `:fragment`. | {} | +| ``quantize.query`` | Hash containing options for query portion of URL quantization. May include `:show` or `:exclude`. See options below. Option must be nested inside the `quantize` option. | {} | +| ``quantize.query.show`` | Defines which values should always be shown. Shows no values by default. May be an Array of strings, or `:all` to show all values. Option must be nested inside the `query` option. | ``nil`` | +| ``quantize.query.exclude`` | Defines which values should be removed entirely. Excludes nothing by default. May be an Array of strings, or `:all` to remove the query string entirely. Option must be nested inside the `query` option. | ``nil`` | +| ``quantize.fragment`` | Defines behavior for URL fragments. Removes fragments by default. May be `:show` to show URL fragments. Option must be nested inside the `quantize` option. | ``nil`` | +| ``application`` | Your Rack application. Necessary for enabling middleware resource names. | ``nil`` | +| ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | -### Dalli +**Configuring URL quantization behavior** -Dalli integration will trace all calls to your ``memcached`` server: +```ruby +Datadog.configure do |c| + # Default behavior: all values are quantized, fragment is removed. + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id&sort_by + # http://example.com/path?categories[]=1&categories[]=2 --> http://example.com/path?categories[] - require 'dalli' - require 'ddtrace' + # Show values for any query string parameter matching 'category_id' exactly + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id=1&sort_by + c.use :rack, quantize: { query: { show: ['category_id'] } } - Datadog.configure do |c| - c.use :dalli, service_name: 'dalli' - end + # Show all values for all query string parameters + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id=1&sort_by=asc + c.use :rack, quantize: { query: { show: :all } } + + # Totally exclude any query string parameter matching 'sort_by' exactly + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id + c.use :rack, quantize: { query: { exclude: ['sort_by'] } } + + # Remove the query string entirely + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path + c.use :rack, quantize: { query: { exclude: :all } } + + # Show URL fragments + # http://example.com/path?category_id=1&sort_by=asc#featured --> http://example.com/path?category_id&sort_by#featured + c.use :rack, quantize: { fragment: :show } +end +``` - client = Dalli::Client.new('localhost:11211', options) - client.set('abc', 123) +### Rails + +The Rails integration will trace requests, database calls, templates rendering and cache read/write/delete operations. The integration makes use of the Active Support Instrumentation, listening to the Notification API so that any operation instrumented by the API is traced. + +To enable the Rails auto instrumentation, create an initializer file in your ``config/initializers`` folder: + +```ruby +# config/initializers/datadog-tracer.rb + +Datadog.configure do |c| + c.use :rails, options +end +``` Where `options` is an optional `Hash` that accepts the following parameters: | Key | Description | Default | | --- | --- | --- | -| ``service_name`` | Service name used for `dalli` instrumentation | memcached | +| ``service_name`` | Service name used when tracing application requests (on the `rack` level) | ```` (inferred from your Rails application namespace) | +| ``controller_service`` | Service name used when tracing a Rails action controller | ``-controller`` | +| ``cache_service`` | Cache service name used when tracing cache activity | ``-cache`` | +| ``database_service`` | Database service name used when tracing database activity | ``-`` | +| ``exception_controller`` | Class or Module which identifies a custom exception controller class. Tracer provides improved error behavior when it can identify custom exception controllers. By default, without this option, it 'guesses' what a custom exception controller looks like. Providing this option aids this identification. | ``nil`` | +| ``distributed_tracing`` | Enables [distributed tracing](#distributed-tracing) so that this service trace is connected with a trace of another service if tracing headers are received | `false` | +| ``middleware_names`` | Enables any short-circuited middleware requests to display the middleware name as resource for the trace. | `false` | +| ``template_base_path`` | Used when the template name is parsed. If you don't store your templates in the ``views/`` folder, you may need to change this value | ``views/`` | +| ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | ### Redis The Redis integration will trace simple calls as well as pipelines. - require 'redis' - require 'ddtrace' +```ruby +require 'redis' +require 'ddtrace' - Datadog.configure do |c| - c.use :redis, service_name: 'redis' - end +Datadog.configure do |c| + c.use :redis, service_name: 'redis' +end - # now do your Redis stuff, eg: - redis = Redis.new - redis.set 'foo', 'bar' # traced! +# now do your Redis stuff, eg: +redis = Redis.new +redis.set 'foo', 'bar' # traced! +``` Where `options` is an optional `Hash` that accepts the following parameters: @@ -460,25 +666,56 @@ Where `options` is an optional `Hash` that accepts the following parameters: You can also set *per-instance* configuration as it follows: - customer_cache = Redis.new - invoice_cache = Redis.new +```ruby +customer_cache = Redis.new +invoice_cache = Redis.new + +Datadog.configure(customer_cache, service_name: 'customer-cache') +Datadog.configure(invoice_cache, service_name: invoice-cache') + +customer_cache.get(...) # traced call will belong to `customer-cache` service +invoice_cache.get(...) # traced call will belong to `invoice-cache` service +``` + +### Resque + +The Resque integration uses Resque hooks that wraps the ``perform`` method. +To add tracing to a Resque job, simply do as follows: + +```ruby +require 'ddtrace' + +class MyJob + def self.perform(*args) + # do_something + end +end - Datadog.configure(customer_cache, service_name: 'customer-cache') - Datadog.configure(invoice_cache, service_name: invoice-cache') +Datadog.configure do |c| + c.use :resque, options +end +``` - customer_cache.get(...) # traced call will belong to `customer-cache` service - invoice_cache.get(...) # traced call will belong to `invoice-cache` service +Where `options` is an optional `Hash` that accepts the following parameters: + +| Key | Description | Default | +| --- | --- | --- | +| ``service_name`` | Service name used for `resque` instrumentation | resque | +| ``workers`` | An array including all worker classes you want to trace (eg ``[MyJob]``) | ``[]`` | ### Sidekiq -The Sidekiq integration is a server-side middleware which will trace job -executions. You can enable it through `Datadog.configure`: +The Sidekiq integration is a server-side middleware which will trace job executions. - require 'ddtrace' +You can enable it through `Datadog.configure`: - Datadog.configure do |c| - c.use :sidekiq, options - end +```ruby +require 'ddtrace' + +Datadog.configure do |c| + c.use :sidekiq, options +end +``` Where `options` is an optional `Hash` that accepts the following parameters: @@ -486,42 +723,50 @@ Where `options` is an optional `Hash` that accepts the following parameters: | --- | --- | --- | | ``service_name`` | Service name used for `sidekiq` instrumentation | sidekiq | -### Resque +### Sinatra -The Resque integration uses Resque hooks that wraps the ``perform`` method. -To add tracing to a Resque job, simply do as follows: +The Sinatra integration traces requests and template rendering. - require 'ddtrace' +To start using the tracing client, make sure you import ``ddtrace`` and ``ddtrace/contrib/sinatra/tracer`` after +either ``sinatra`` or ``sinatra/base``: - class MyJob - def self.perform(*args) - # do_something - end - end +```ruby +require 'sinatra' +require 'ddtrace' +require 'ddtrace/contrib/sinatra/tracer' - Datadog.configure do |c| - c.use :resque, options - end +Datadog.configure do |c| + c.use :sinatra, options +end + +get '/' do + 'Hello world!' +end +``` Where `options` is an optional `Hash` that accepts the following parameters: | Key | Description | Default | | --- | --- | --- | -| ``service_name`` | Service name used for `resque` instrumentation | resque | -| ``workers`` | An array including all worker classes you want to trace (eg ``[MyJob]``) | ``[]`` | +| ``service_name`` | Service name used for `sinatra` instrumentation | sinatra | +| ``resource_script_names`` | Prepend resource names with script name | ``false`` | +| ``distributed_tracing`` | Enables [distributed tracing](#distributed-tracing) so that this service trace is connected with a trace of another service if tracing headers are received | `false` | +| ``tracer`` | A ``Datadog::Tracer`` instance used to instrument the application. Usually you don't need to set that. | ``Datadog.tracer`` | -### SuckerPunch +### Sucker Punch The `sucker_punch` integration traces all scheduled jobs: - require 'ddtrace' +```ruby +require 'ddtrace' - Datadog.configure do |c| - c.use :sucker_punch, options - end +Datadog.configure do |c| + c.use :sucker_punch, options +end - # the execution of this job is traced - LogJob.perform_async('login') +# the execution of this job is traced +LogJob.perform_async('login') +``` Where `options` is an optional `Hash` that accepts the following parameters: @@ -529,113 +774,77 @@ Where `options` is an optional `Hash` that accepts the following parameters: | --- | --- | --- | | ``service_name`` | Service name used for `sucker_punch` instrumentation | sucker_punch | -## Advanced usage +## Advanced configuration -### Configure the tracer +### Tracer settings To change the default behavior of the Datadog tracer, you can provide custom options inside the `Datadog.configure` block as in: - # config/initializers/datadog-tracer.rb +```ruby +# config/initializers/datadog-tracer.rb - Datadog.configure do |c| - c.tracer option_name: option_value, ... - end +Datadog.configure do |c| + c.tracer option_name: option_value, ... +end +``` Available options are: -* ``enabled``: defines if the ``tracer`` is enabled or not. If set to ``false`` the code could be still instrumented + - ``enabled``: defines if the ``tracer`` is enabled or not. If set to ``false`` the code could be still instrumented because of other settings, but no spans are sent to the local trace agent. -* ``debug``: set to true to enable debug logging. -* ``hostname``: set the hostname of the trace agent. -* ``port``: set the port the trace agent is listening on. -* ``env``: set the environment. Rails users may set it to ``Rails.env`` to use their application settings. -* ``tags``: set global tags that should be applied to all spans. Defaults to an empty hash -* ``log``: defines a custom logger. + - ``debug``: set to true to enable debug logging. + - ``hostname``: set the hostname of the trace agent. + - ``port``: set the port the trace agent is listening on. + - ``env``: set the environment. Rails users may set it to ``Rails.env`` to use their application settings. + - ``tags``: set global tags that should be applied to all spans. Defaults to an empty hash + - ``log``: defines a custom logger. -#### Using a custom logger +#### Custom logging -By default, all logs are processed by the default Ruby logger. -Typically, when using Rails, you should see the messages in your application log file. -Datadog client log messages are marked with ``[ddtrace]`` so you should be able -to isolate them from other messages. - -Additionally, it is possible to override the default logger and replace it by a -custom one. This is done using the ``log`` attribute of the tracer. - - f = File.new("my-custom.log", "w+") # Log messages should go there - Datadog.configure do |c| - c.tracer log: Logger.new(f) # Overriding the default tracer - end +By default, all logs are processed by the default Ruby logger. When using Rails, you should see the messages in your application log file. - Datadog::Tracer.log.info { "this is typically called by tracing code" } +Datadog client log messages are marked with ``[ddtrace]`` so you should be able to isolate them from other messages. -### Manual Instrumentation +Additionally, it is possible to override the default logger and replace it by a custom one. This is done using the ``log`` attribute of the tracer. -If you aren't using a supported framework instrumentation, you may want to to manually instrument your code. -Adding tracing to your code is very simple. As an example, let’s imagine we have a web server and we want -to trace requests to the home page: +```ruby +f = File.new("my-custom.log", "w+") # Log messages should go there +Datadog.configure do |c| + c.tracer log: Logger.new(f) # Overriding the default tracer +end - require 'ddtrace' - require 'sinatra' - require 'active_record' - - # a generic tracer that you can use across your application - tracer = Datadog.tracer - - get '/' do - tracer.trace('web.request') do |span| - # set some span metadata - span.service = 'my-web-site' - span.resource = '/' - - # trace the activerecord call - tracer.trace('posts.fetch') do - @posts = Posts.order(created_at: :desc).limit(10) - end - - # add some attributes and metrics - span.set_tag('http.method', request.request_method) - span.set_tag('posts.count', @posts.length) - - # trace the template rendering - tracer.trace('template.render') do - erb :index - end - end - end +Datadog::Tracer.log.info { "this is typically called by tracing code" } +``` ### Environment and tags -By default, the trace agent (not this library, but the program running in -the background collecting data from various clients) uses the tags -set in the agent config file, see our -[environments tutorial](https://app.datadoghq.com/apm/docs/tutorials/environments) for details. +By default, the trace agent (not this library, but the program running in the background collecting data from various clients) uses the tags set in the agent config file, see our [environments tutorial](https://app.datadoghq.com/apm/docs/tutorials/environments) for details. These values can be overridden at the tracer level: - Datadog.configure do |c| - c.tracer tags: { 'env' => 'prod' } - end +```ruby +Datadog.configure do |c| + c.tracer tags: { 'env' => 'prod' } +end +``` -This enables you to set this value on a per tracer basis, so you can have -for example several applications reporting for different environments on the same host. +This enables you to set this value on a per tracer basis, so you can have for example several applications reporting for different environments on the same host. -Ultimately, tags can be set per span, but `env` should typically be the same -for all spans belonging to a given trace. +Ultimately, tags can be set per span, but `env` should typically be the same for all spans belonging to a given trace. ### Sampling -`ddtrace` can perform trace sampling. While the trace agent already samples -traces to reduce bandwidth usage, client sampling reduces performance -overhead. +`ddtrace` can perform trace sampling. While the trace agent already samples traces to reduce bandwidth usage, client sampling reduces performance overhead. `Datadog::RateSampler` samples a ratio of the traces. For example: - # Sample rate is between 0 (nothing sampled) to 1 (everything sampled). - sampler = Datadog::RateSampler.new(0.5) # sample 50% of the traces - Datadog.configure do |c| - c.tracer sampler: sampler - end +```ruby +# Sample rate is between 0 (nothing sampled) to 1 (everything sampled). +sampler = Datadog::RateSampler.new(0.5) # sample 50% of the traces +Datadog.configure do |c| + c.tracer sampler: sampler +end +``` #### Priority sampling @@ -643,12 +852,12 @@ Priority sampling consists in deciding if a trace will be kept by using a priori The sampler can set the priority to the following values: -* `Datadog::Ext::Priority::AUTO_REJECT`: the sampler automatically decided to reject the trace. -* `Datadog::Ext::Priority::AUTO_KEEP`: the sampler automatically decided to keep the trace. + - `Datadog::Ext::Priority::AUTO_REJECT`: the sampler automatically decided to reject the trace. + - `Datadog::Ext::Priority::AUTO_KEEP`: the sampler automatically decided to keep the trace. For now, priority sampling is disabled by default. Enabling it ensures that your sampled distributed traces will be complete. To enable the priority sampling: -```rb +```ruby Datadog.configure do |c| c.tracer priority_sampling: true end @@ -658,19 +867,14 @@ Once enabled, the sampler will automatically assign a priority of 0 or 1 to trac You can also set this priority manually to either drop a non-interesting trace or to keep an important one. For that, set the `context#sampling_priority` to: -* `Datadog::Ext::Priority::USER_REJECT`: the user asked to reject the trace. -* `Datadog::Ext::Priority::USER_KEEP`: the user asked to keep the trace. + - `Datadog::Ext::Priority::USER_REJECT`: the user asked to reject the trace. + - `Datadog::Ext::Priority::USER_KEEP`: the user asked to keep the trace. -When not using [distributed tracing](#Distributed_Tracing), you may change the priority at any time, -as long as the trace is not finished yet. -But it has to be done before any context propagation (fork, RPC calls) to be effective in a distributed context. -Changing the priority after context has been propagated causes different parts of a distributed trace -to use different priorities. Some parts might be kept, some parts might be rejected, -and this can cause the trace to be partially stored and remain incomplete. +When not using [distributed tracing](#distributed-tracing), you may change the priority at any time, as long as the trace is not finished yet. But it has to be done before any context propagation (fork, RPC calls) to be effective in a distributed context. Changing the priority after context has been propagated causes different parts of a distributed trace to use different priorities. Some parts might be kept, some parts might be rejected, and this can cause the trace to be partially stored and remain incomplete. If you change the priority, we recommend you do it as soon as possible, when the root span has just been created. -```rb +```ruby # Indicate to reject the trace span.context.sampling_priority = Datadog::Ext::Priority::USER_REJECT @@ -682,14 +886,16 @@ span.context.sampling_priority = Datadog::Ext::Priority::USER_KEEP To trace requests across hosts, the spans on the secondary hosts must be linked together by setting ``trace_id`` and ``parent_id``: - def request_on_secondary_host(parent_trace_id, parent_span_id) - tracer.trace('web.request') do |span| - span.parent_id = parent_span_id - span.trace_id = parent_trace_id +```ruby +def request_on_secondary_host(parent_trace_id, parent_span_id) + tracer.trace('web.request') do |span| + span.parent_id = parent_span_id + span.trace_id = parent_trace_id - # perform user code - end + # perform user code end +end +``` Users can pass along the ``parent_trace_id`` and ``parent_span_id`` via whatever method best matches the RPC framework. @@ -697,103 +903,109 @@ Below is an example using Net/HTTP and Sinatra, where we bypass the integrations On the client: - require 'net/http' - require 'ddtrace' +```ruby +require 'net/http' +require 'ddtrace' - uri = URI('http://localhost:4567/') +uri = URI('http://localhost:4567/') - Datadog.tracer.trace('web.call') do |span| - req = Net::HTTP::Get.new(uri) - req['x-datadog-trace-id'] = span.trace_id.to_s - req['x-datadog-parent-id'] = span.span_id.to_s +Datadog.tracer.trace('web.call') do |span| + req = Net::HTTP::Get.new(uri) + req['x-datadog-trace-id'] = span.trace_id.to_s + req['x-datadog-parent-id'] = span.span_id.to_s - response = Net::HTTP.start(uri.hostname, uri.port) do |http| - http.request(req) - end + response = Net::HTTP.start(uri.hostname, uri.port) do |http| + http.request(req) + end - puts response.body - end + puts response.body +end +``` On the server: - require 'sinatra' - require 'ddtrace' +```ruby +require 'sinatra' +require 'ddtrace' - get '/' do - parent_trace_id = request.env['HTTP_X_DATADOG_TRACE_ID'] - parent_span_id = request.env['HTTP_X_DATADOG_PARENT_ID'] +get '/' do + parent_trace_id = request.env['HTTP_X_DATADOG_TRACE_ID'] + parent_span_id = request.env['HTTP_X_DATADOG_PARENT_ID'] - Datadog.tracer.trace('web.work') do |span| - if parent_trace_id && parent_span_id - span.trace_id = parent_trace_id.to_i - span.parent_id = parent_span_id.to_i - end + Datadog.tracer.trace('web.work') do |span| + if parent_trace_id && parent_span_id + span.trace_id = parent_trace_id.to_i + span.parent_id = parent_span_id.to_i + end - 'Hello world!' - end - end + 'Hello world!' + end +end +``` -[Rack](#Rack) and [Net/HTTP](#Net_HTTP) have experimental support for this, they -can send and receive these headers automatically and tie spans together automatically, -provided you pass a ``:distributed_tracing`` option set to ``true``. +[Rack](#rack) and [Net/HTTP](#nethttp) have experimental support for this, they can send and receive these headers automatically and tie spans together automatically, provided you pass a ``:distributed_tracing`` option set to ``true``. -This is disabled by default. +Distributed tracing is disabled by default. ### Processing Pipeline -Sometimes it might be interesting to intercept `Span` objects before they get -sent upstream. To achieve that, you can hook custom *processors* into the -pipeline using the method `Datadog::Pipeline.before_flush`: - - Datadog::Pipeline.before_flush( - # filter the Span if the given block evaluates true - Datadog::Pipeline::SpanFilter.new { |span| span.resource =~ /PingController/ }, - Datadog::Pipeline::SpanFilter.new { |span| span.get_tag('host') == 'localhost' }, - - # alter the Span updating fields or tags - Datadog::Pipeline::SpanProcessor.new { |span| span.resource.gsub!(/password=.*/, '') } - ) +Some applications might require that traces be altered or filtered out before they are sent upstream. The processing pipeline allows users to create *processors* to define such behavior. -For more information, please refer to this [link](https://github.com/DataDog/dd-trace-rb/pull/214). +Processors can be any object that responds to `#call` accepting `trace` as an argument (which is an `Array` of `Datadog::Span`s.) -### Supported Versions +For example: -#### Ruby interpreters - -The Datadog Trace Client has been tested with the following Ruby versions: +```ruby +lambda_processor = ->(trace) do + # Processing logic... + trace +end -* Ruby MRI 1.9.1 (experimental) -* Ruby MRI 1.9.3 -* Ruby MRI 2.0 -* Ruby MRI 2.1 -* Ruby MRI 2.2 -* Ruby MRI 2.3 -* Ruby MRI 2.4 -* JRuby 9.1.5 (experimental) +class MyCustomProcessor + def call(trace) + # Processing logic... + trace + end +end +custom_processor = MyFancyProcessor.new +``` -Other versions aren't yet officially supported. +`#call` blocks of processors *must* return the `trace` object; this return value will be passed to the next processor in the pipeline. -#### Ruby on Rails versions +These processors must then be added to the pipeline via `Datadog::Pipeline.before_flush`: -The supported versions are: +```ruby +Datadog::Pipeline.before_flush(lambda_processor, custom_processor) +``` -* Rails 3.2 (MRI interpreter, JRuby is experimental) -* Rails 4.2 (MRI interpreter, JRuby is experimental) -* Rails 5.0 (MRI interpreter) +You can also define processors using the short-hand block syntax for `Datadog::Pipeline.before_flush`: -The currently supported web server are: -* Puma 2.16+ and 3.6+ -* Unicorn 4.8+ and 5.1+ -* Passenger 5.0+ +```ruby +Datadog::Pipeline.before_flush do |trace| + trace.delete_if { |span| span.name =~ /forbidden/ } +end +``` -#### Sinatra versions +#### Filtering -Currently we are supporting Sinatra >= 1.4.0. +You can use the `Datadog::Pipeline::SpanFilter` processor to remove spans, when the block evaluates as truthy: -#### Sidekiq versions +```ruby +Datadog::Pipeline.before_flush( + # Remove spans that match a particular resource + Datadog::Pipeline::SpanFilter.new { |span| span.resource =~ /PingController/ }, + # Remove spans that are trafficked to localhost + Datadog::Pipeline::SpanFilter.new { |span| span.get_tag('host') == 'localhost' } +) +``` -Currently we are supporting Sidekiq >= 4.0.0. +#### Processing -### Terminology +You can use the `Datadog::Pipeline::SpanProcessor` processor to modify spans: -If you need more context about the terminology used in the APM, take a look at the [official documentation](https://docs.datadoghq.com/tracing/terminology/). +```ruby +Datadog::Pipeline.before_flush( + # Strip matching text from the resource field + Datadog::Pipeline::SpanProcessor.new { |span| span.resource.gsub!(/password=.*/, '') } +) +``` From a22d48ff924e8cca1de568148a03bf0d5404a95a Mon Sep 17 00:00:00 2001 From: "Christian Mauduit (DataDog)" Date: Wed, 8 Nov 2017 16:26:52 -0500 Subject: [PATCH 47/72] [tracer] reduce memory usage on high-cardinality traces For big traces (typically, long-running traces with one enclosing span and many sub-spans, possibly several thousands) the library could keep everything in memory waiting for an hypothetical flush. This patch partially flushes consistent parts of traces, so that they don't fill up the RAM. --- lib/ddtrace/context.rb | 71 ++++++- lib/ddtrace/context_flush.rb | 132 ++++++++++++ lib/ddtrace/tracer.rb | 23 ++- test/context_flush_test.rb | 376 +++++++++++++++++++++++++++++++++++ test/context_test.rb | 98 ++++++++- test/span_test.rb | 1 - 6 files changed, 694 insertions(+), 7 deletions(-) create mode 100644 lib/ddtrace/context_flush.rb create mode 100644 test/context_flush_test.rb diff --git a/lib/ddtrace/context.rb b/lib/ddtrace/context.rb index bd6ec141292..8b852a6baf3 100644 --- a/lib/ddtrace/context.rb +++ b/lib/ddtrace/context.rb @@ -13,10 +13,19 @@ module Datadog # \Context, it will be related to the original trace. # # This data structure is thread-safe. + # rubocop:disable Metrics/ClassLength class Context + # 100k spans is about a 100Mb footprint + DEFAULT_MAX_LENGTH = 100_000 + + attr_reader :max_length + # Initialize a new thread-safe \Context. def initialize(options = {}) @mutex = Mutex.new + # max_length is the amount of spans above which, for a given trace, + # the context will simply drop and ignore spans, avoiding high memory usage. + @max_length = options.fetch(:max_length, DEFAULT_MAX_LENGTH) reset(options) end @@ -78,6 +87,16 @@ def set_current_span(span) # Add a span to the context trace list, keeping it as the last active span. def add_span(span) @mutex.synchronize do + # If hitting the hard limit, just drop spans. This is really a rare case + # as it means despite the soft limit, the hard limit is reached, so the trace + # by default has 10000 spans, all of which belong to unfinished parts of a + # larger trace. This is a catch-all to reduce global memory usage. + if @max_length > 0 && @trace.length >= @max_length + Datadog::Tracer.log.debug("context full, ignoring span #{span.name}") + # Detach the span from any context, it's being dropped and ignored. + span.context = nil + return + end set_current_span(span) @trace << span span.context = self @@ -135,14 +154,16 @@ def sampled? # This operation is thread-safe. def get @mutex.synchronize do - return nil, nil unless check_finished_spans - trace = @trace sampled = @sampled + attach_sampling_priority if sampled && @sampling_priority + # still return sampled attribute, even if context is not finished + return nil, sampled unless check_finished_spans() + reset - return trace, sampled + [trace, sampled] end end @@ -161,6 +182,50 @@ def attach_sampling_priority ) end + # Return the start time of the root span, or nil if there are no spans or this is undefined. + def start_time + @mutex.synchronize do + return nil if @trace.empty? + @trace[0].start_time + end + end + + # Return the length of the current trace held by this context. + def length + @mutex.synchronize do + @trace.length + end + end + + # Iterate on each span within the trace. This is thread safe. + def each_span + @mutex.synchronize do + @trace.each do |span| + yield span + end + end + end + + # Delete any span matching the condition. This is thread safe. + def delete_span_if + @mutex.synchronize do + @trace.delete_if do |span| + finished = span.finished? + delete_span = yield span + if delete_span + # We need to detach the span from the context, else, some code + # finishing it afterwards would mess up with the number of + # finished_spans and possibly cause other side effects. + span.context = nil + # Acknowledge there's one span less to finish, if needed. + # It's very important to keep this balanced. + @finished_spans -= 1 if finished + end + delete_span + end + end + end + private :reset private :check_finished_spans private :set_current_span diff --git a/lib/ddtrace/context_flush.rb b/lib/ddtrace/context_flush.rb new file mode 100644 index 00000000000..b44792d1e3d --- /dev/null +++ b/lib/ddtrace/context_flush.rb @@ -0,0 +1,132 @@ +require 'set' + +require 'ddtrace/context' + +module Datadog + # \ContextFlush is used to cap context size and avoid it using too much memory. + # It performs memory flushes when required. + class ContextFlush + # by default, soft and hard limits are the same + DEFAULT_MAX_SPANS_BEFORE_PARTIAL_FLUSH = Datadog::Context::DEFAULT_MAX_LENGTH + # by default, never do a partial flush + DEFAULT_MIN_SPANS_BEFORE_PARTIAL_FLUSH = Datadog::Context::DEFAULT_MAX_LENGTH + # timeout should be lower than the trace agent window + DEFAULT_PARTIAL_FLUSH_TIMEOUT = 10 + + private_constant :DEFAULT_MAX_SPANS_BEFORE_PARTIAL_FLUSH + private_constant :DEFAULT_MIN_SPANS_BEFORE_PARTIAL_FLUSH + private_constant :DEFAULT_PARTIAL_FLUSH_TIMEOUT + + def initialize(options = {}) + # max_spans_before_partial_flush is the amount of spans collected before + # the context starts to partially flush parts of traces. With a setting of 10k, + # the memory overhead is about 10Mb per thread/context (depends on spans metadata, + # this is just an order of magnitude). + @max_spans_before_partial_flush = options.fetch(:max_spans_before_partial_flush, + DEFAULT_MAX_SPANS_BEFORE_PARTIAL_FLUSH) + # min_spans_before_partial_flush is the minimum number of spans required + # for a partial flush to happen on a timeout. This is to prevent partial flush + # of traces which last a very long time but yet have few spans. + @min_spans_before_partial_flush = options.fetch(:min_spans_before_partial_flush, + DEFAULT_MIN_SPANS_BEFORE_PARTIAL_FLUSH) + # partial_flush_timeout is the limit (in seconds) above which the context + # considers flushing parts of the trace. Partial flushes should not be done too + # late else the agent rejects them with a "too far in the past" error. + @partial_flush_timeout = options.fetch(:partial_flush_timeout, + DEFAULT_PARTIAL_FLUSH_TIMEOUT) + @partial_traces = [] + end + + def add_children(m, spans, ids, leaf) + spans << leaf + ids.add(leaf.span_id) + + if m[leaf.span_id] + m[leaf.span_id].each do |sub| + add_children(m, spans, ids, sub) + end + end + end + + def partial_traces(context) + # 1st step, taint all parents of an unfinished span as unflushable + unflushable_ids = Set.new + + context.each_span do |span| + next if span.finished? || unflushable_ids.include?(span.span_id) + unflushable_ids.add span.span_id + while span.parent + span = span.parent + unflushable_ids.add span.span_id + end + end + + # 2nd step, find all spans which are at the border between flushable and unflushable + # Along the road, collect a reverse-tree which allows direct walking from parents to + # children but only for the ones we're interested it. + roots = [] + children_map = {} + context.each_span do |span| + # There's no point in trying to put the real root in those partial roots, if + # it's flushable, the default algorithm would figure way more quickly. + if span.parent && !unflushable_ids.include?(span.span_id) + if unflushable_ids.include?(span.parent.span_id) + # span is flushable but is parent is not + roots << span + else + # span is flushable and its parent is too, build the reverse + # parent to child map for this one, it will be useful + children_map[span.parent.span_id] ||= [] + children_map[span.parent.span_id] << span + end + end + end + + # 3rd step, find all children, as this can be costly, only perform it for partial roots + partial_traces = [] + all_ids = Set.new + roots.each do |root| + spans = [] + add_children(children_map, spans, all_ids, root) + partial_traces << spans + end + + return [nil, nil] if partial_traces.empty? + [partial_traces, all_ids] + end + + def partial_flush(context) + traces, flushed_ids = partial_traces(context) + return nil unless traces && flushed_ids + + # We need to reject by span ID and not by value, because a span + # value may be altered (typical example: it's finished by some other thread) + # since we lock only the context, not all the spans which belong to it. + context.delete_span_if { |span| flushed_ids.include? span.span_id } + traces + end + + # Performs an operation which each partial trace it can get from the context. + def each_partial_trace(context) + start_time = context.start_time + length = context.length + # Stop and do not flush anything if there are not enough spans. + return if length <= @min_spans_before_partial_flush + # If there are enough spans, but not too many, check for start time. + # If timeout is not given or 0, then wait + return if length <= @max_spans_before_partial_flush && + (@partial_flush_timeout.nil? || @partial_flush_timeout <= 0 || + (start_time && start_time > Time.now.utc - @partial_flush_timeout)) + # Here, either the trace is old or we have too many spans, flush it. + traces = partial_flush(context) + return unless traces + traces.each do |trace| + yield trace + end + end + + private :add_children + private :partial_traces + private :partial_flush + end +end diff --git a/lib/ddtrace/tracer.rb b/lib/ddtrace/tracer.rb index fb27584cee8..eab03865157 100644 --- a/lib/ddtrace/tracer.rb +++ b/lib/ddtrace/tracer.rb @@ -5,6 +5,7 @@ require 'ddtrace/span' require 'ddtrace/context' +require 'ddtrace/context_flush' require 'ddtrace/provider' require 'ddtrace/logger' require 'ddtrace/writer' @@ -97,6 +98,8 @@ def initialize(options = {}) @provider = options.fetch(:context_provider, Datadog::DefaultContextProvider.new) @provider ||= Datadog::DefaultContextProvider.new # @provider should never be nil + @context_flush = Datadog::ContextFlush.new(options) + @mutex = Mutex.new @services = {} @tags = {} @@ -117,8 +120,13 @@ def configure(options = {}) enabled = options.fetch(:enabled, nil) hostname = options.fetch(:hostname, nil) port = options.fetch(:port, nil) + + # Those are rare "power-user" options. sampler = options.fetch(:sampler, nil) priority_sampling = options[:priority_sampling] + max_spans_before_partial_flush = options.fetch(:max_spans_before_partial_flush, nil) + min_spans_before_partial_flush = options.fetch(:max_spans_before_partial_flush, nil) + partial_flush_timeout = options.fetch(:partial_flush_timeout, nil) @enabled = enabled unless enabled.nil? @sampler = sampler unless sampler.nil? @@ -130,6 +138,10 @@ def configure(options = {}) @writer.transport.hostname = hostname unless hostname.nil? @writer.transport.port = port unless port.nil? + + @context_flush = Datadog::ContextFlush.new(options) unless min_spans_before_partial_flush.nil? && + max_spans_before_partial_flush.nil? && + partial_flush_timeout.nil? end # Set the information about the given service. A valid example is: @@ -295,8 +307,15 @@ def record(context) context = context.context if context.is_a?(Datadog::Span) return if context.nil? trace, sampled = context.get - ready = !trace.nil? && !trace.empty? && sampled - write(trace) if ready + if sampled + if trace.nil? || trace.empty? + @context_flush.each_partial_trace(context) do |t| + write(t) + end + else + write(trace) + end + end end # Return the current active span or +nil+. diff --git a/test/context_flush_test.rb b/test/context_flush_test.rb new file mode 100644 index 00000000000..947ac531796 --- /dev/null +++ b/test/context_flush_test.rb @@ -0,0 +1,376 @@ +require 'helper' +require 'ddtrace/tracer' +require 'ddtrace/context_flush' + +class ContextFlushEachTest < Minitest::Test + def test_each_partial_trace_typical_not_enough_traces + tracer = get_test_tracer + context_flush = Datadog::ContextFlush.new + context = tracer.call_context + + context_flush.each_partial_trace(context) do |_t| + flunk('nothing should be partially flushed, no spans') + end + + # the plan: + # + # root-------------. + # | \______ \ + # | \ \ + # child1 child3 child4 + # | | \_____ + # | | \ + # child2 child5 child6 + + tracer.trace('root') do + tracer.trace('child1') do + tracer.trace('child2') do + end + end + tracer.trace('child3') do + # finished spans are CAPITALIZED + # + # root + # | \______ + # | \ + # CHILD1 child3 + # | + # | + # CHILD2 + context_flush.each_partial_trace(context) do |t| + flunk("nothing should be partially flushed, got: #{t}") + end + end + tracer.trace('child4') do + tracer.trace('child5') do + end + tracer.trace('child6') do + end + end + # finished spans are CAPITALIZED + # + # root-------------. + # | \______ \ + # | \ \ + # CHILD1 CHILD3 CHILD4 + # | | \_____ + # | | \ + # CHILD2 CHILD5 CHILD6 + context_flush.each_partial_trace(context) do |t| + flunk("nothing should be partially flushed, got: #{t}") + end + end + + context_flush.each_partial_trace(context) do |t| + flunk("nothing should be partially flushed, got: #{t}") + end + + assert_equal(0, context.length, 'everything should be written by now') + end + + def test_each_partial_trace_typical + tracer = get_test_tracer + context_flush = Datadog::ContextFlush.new(min_spans_before_partial_flush: 1, + max_spans_before_partial_flush: 1) + context = tracer.call_context + + # the plan: + # + # root-------------. + # | \______ \ + # | \ \ + # child1 child3 child4 + # | | \_____ + # | | \ + # child2 child5 child6 + + action12 = Minitest::Mock.new + action12.expect(:call_with_names, nil, [%w[child1 child2].to_set]) + action3456 = Minitest::Mock.new + action3456.expect(:call_with_names, nil, [['child3'].to_set]) + action3456.expect(:call_with_names, nil, [%w[child4 child5 child6].to_set]) + + tracer.trace('root') do + tracer.trace('child1') do + tracer.trace('child2') do + end + end + tracer.trace('child3') do + # finished spans are CAPITALIZED + # + # root + # | \______ + # | \ + # CHILD1 child3 + # | + # | + # CHILD2 + context_flush.each_partial_trace(context) do |t| + action12.call_with_names(t.map(&:name).to_set) + end + end + tracer.trace('child4') do + tracer.trace('child5') do + end + tracer.trace('child6') do + end + end + # finished spans are CAPITALIZED + # + # root-------------. + # \______ \ + # \ \ + # CHILD3 CHILD4 + # | \_____ + # | \ + # CHILD5 CHILD6 + context_flush.each_partial_trace(context) do |t| + action3456.call_with_names(t.map(&:name).to_set) + end + end + + action12.verify + action3456.verify + + assert_equal(0, context.length, 'everything should be written by now') + end + + # rubocop:disable Metrics/MethodLength + def test_each_partial_trace_mixed + tracer = get_test_tracer + context_flush = Datadog::ContextFlush.new(min_spans_before_partial_flush: 1, + max_spans_before_partial_flush: 1) + context = tracer.call_context + + # the plan: + # + # root + # | \______ + # | \ + # child1 child5 + # | + # | + # child2 + # | \______ + # | \ + # child3 child6 + # | | + # | | + # child4 child7 + + action345 = Minitest::Mock.new + action345.expect(:call_with_names, nil, [%w[child3 child4].to_set]) + action345.expect(:call_with_names, nil, [%w[child5].to_set]) + + root = tracer.start_span('root', child_of: context) + child1 = tracer.start_span('child1', child_of: root) + child2 = tracer.start_span('child2', child_of: child1) + child3 = tracer.start_span('child3', child_of: child2) + child4 = tracer.start_span('child4', child_of: child3) + child5 = tracer.start_span('child5', child_of: root) + child6 = tracer.start_span('child6', child_of: child2) + child7 = tracer.start_span('child7', child_of: child6) + + context_flush.each_partial_trace(context) do |_t| + context_flush.each_partial_trace(context) do |_t| + flunk('nothing should be partially flushed, no span is finished') + end + end + + assert_equal(8, context.length) + + [root, child1, child3, child6].each do |span| + span.finish + context_flush.each_partial_trace(context) do |t| + flunk("nothing should be partially flushed, got: #{t}") + end + end + + # finished spans are CAPITALIZED + # + # ROOT + # | \______ + # | \ + # CHILD1 child5 + # | + # | + # child2 + # | \______ + # | \ + # CHILD3 CHILD6 + # | | + # | | + # child4 child7 + + child2.finish + + context_flush.each_partial_trace(context) do |t| + flunk("nothing should be partially flushed, got: #{t}") + end + + # finished spans are CAPITALIZED + # + # ROOT + # | \______ + # | \ + # CHILD1 child5 + # | + # | + # CHILD2 + # | \______ + # | \ + # CHILD3 CHILD6 + # | | + # | | + # child4 child7 + + child4.finish + child5.finish + + # finished spans are CAPITALIZED + # + # ROOT + # | \______ + # | \ + # CHILD1 CHILD5 + # | + # | + # CHILD2 + # | \______ + # | \ + # CHILD3 CHILD6 + # | | + # | | + # CHILD4 child7 + + context_flush.each_partial_trace(context) do |t| + action345.call_with_names(t.map(&:name).to_set) + end + + child7.finish + + context_flush.each_partial_trace(context) do |t| + flunk("nothing should be partially flushed, got: #{t}") + end + + assert_equal(0, context.length, 'everything should be written by now') + end +end + +module Datadog + class Tracer + attr_accessor :context_flush + end +end + +class ContextFlushPartialTest < Minitest::Test + MIN_SPANS = 10 + MAX_SPANS = 100 + TIMEOUT = 60 # make this very high to reduce test flakiness (1 minute here) + + def get_context_flush + Datadog::ContextFlush.new(min_spans_before_partial_flush: MIN_SPANS, + max_spans_before_partial_flush: MAX_SPANS, + partial_flush_timeout: TIMEOUT) + end + + # rubocop:disable Metrics/AbcSize + def test_partial_caterpillar + tracer = get_test_tracer + context_flush = get_context_flush + tracer.context_flush = context_flush + + write1 = Minitest::Mock.new + expected = [] + MIN_SPANS.times do |i| + expected << "a.#{i}" + end + (MAX_SPANS - MIN_SPANS).times do |i| + expected << "b.#{i}" + end + # We need to sort the values the same way the values will be output by the test transport + expected.sort! + expected.each do |e| + write1.expect(:call_with_name, nil, [e]) + end + + write2 = Minitest::Mock.new + expected = ['root'] + MIN_SPANS.times do |i| + expected << "b.#{i + MAX_SPANS - MIN_SPANS}" + end + # We need to sort the values the same way the values will be output by the test transport + expected.sort! + expected.each do |e| + write2.expect(:call_with_name, nil, [e]) + end + + tracer.trace('root') do + MIN_SPANS.times do |i| + tracer.trace("a.#{i}") do + end + end + spans = tracer.writer.spans() + assert_equal(0, spans.length, 'nothing should be flushed, as max limit is not reached') + MAX_SPANS.times do |i| + tracer.trace("b.#{i}") do + end + end + spans = tracer.writer.spans() + # Let's explain the extra span here, what should happen is: + # - root span is started + # - then 99 spans (10 from 1st batch, 89 from second batch) are put in context + # - then the 101th comes (the 90th from the second batch) and triggers a flush of everything but root span + # - then the last 10 spans from second batch are thrown in, so that's 10 left + the root span + assert_equal(1 + MIN_SPANS, tracer.call_context.length, 'some spans should have been sent') + assert_equal(MAX_SPANS, spans.length) + spans.each do |span| + write1.call_with_name(span.name) + end + write1.verify + end + + spans = tracer.writer.spans() + assert_equal(MIN_SPANS + 1, spans.length) + spans.each do |span| + write2.call_with_name(span.name) + end + write2.verify + end + + # Test the tracer configure args which are forwarded to context flush only. + def test_tracer_configure + tracer = get_test_tracer + + old_context_flush = tracer.context_flush + tracer.configure() + assert_equal(old_context_flush, tracer.context_flush, 'the same context_flush should be reused') + + tracer.configure(min_spans_before_partial_flush: 3, + max_spans_before_partial_flush: 3) + + refute_equal(old_context_flush, tracer.context_flush, 'another context_flush should be have been created') + end + + def test_tracer_hard_limit_overrides_soft_limit + tracer = get_test_tracer + + context = tracer.call_context + tracer.configure(min_spans_before_partial_flush: context.max_length, + max_spans_before_partial_flush: context.max_length, + partial_flush_timeout: 3600) + + n = 1_000_000 + assert_operator(n, :>, context.max_length, 'need to send enough spans') + tracer.trace('root') do + n.times do |_i| + tracer.trace('span.${i}') do + end + spans = tracer.writer.spans() + assert_equal(0, spans.length, 'nothing should be written, soft limit is inhibited') + end + end + spans = tracer.writer.spans() + assert_equal(context.max_length, spans.length, 'size should be capped to hard limit') + end +end diff --git a/test/context_test.rb b/test/context_test.rb index 018e246af51..0d9aebcde7f 100644 --- a/test/context_test.rb +++ b/test/context_test.rb @@ -179,6 +179,7 @@ def test_log_unfinished_spans tracer = get_test_tracer default_log = Datadog::Tracer.log + default_level = Datadog::Tracer.log.level buf = StringIO.new @@ -204,7 +205,7 @@ def test_log_unfinished_spans root.finish() lines = buf.string.lines - assert_equal(3, lines.length, 'there should be 2 log messages') if lines.respond_to? :length + assert_operator(3, :<=, lines.length, 'there should be at least 3 log messages') if lines.respond_to? :length # Test below iterates on lines, this is required for Ruby 1.9 backward compatibility. i = 0 @@ -230,6 +231,7 @@ def test_log_unfinished_spans end Datadog::Tracer.log = default_log + Datadog::Tracer.log.level = default_level end def test_thread_safe @@ -271,6 +273,100 @@ def test_thread_safe assert_nil(ctx.current_span) assert_equal(false, ctx.sampled) end + + def test_length + tracer = get_test_tracer + ctx = Datadog::Context.new + + assert_equal(0, ctx.length) + 10.times do |i| + span = Datadog::Span.new(tracer, "test.op#{i}") + assert_equal(i, ctx.length) + ctx.add_span(span) + assert_equal(i + 1, ctx.length) + ctx.close_span(span) + assert_equal(i + 1, ctx.length) + end + + ctx.get + + assert_equal(0, ctx.length) + end + + def test_start_time + tracer = get_test_tracer + ctx = tracer.call_context + + assert_nil(ctx.start_time) + tracer.trace('test.op') do |span| + assert_equal(span.start_time, ctx.start_time) + end + assert_nil(ctx.start_time) + end + + def test_each_span + span = Datadog::Span.new(nil, 'test.op') + ctx = Datadog::Context.new + ctx.add_span(span) + + action = MiniTest::Mock.new + action.expect(:call_with_name, nil, ['test.op']) + ctx.each_span do |s| + action.call_with_name(s.name) + end + action.verify + end + + def test_delete_span_if + tracer = get_test_tracer + ctx = tracer.call_context + + action = MiniTest::Mock.new + action.expect(:call_with_name, nil, ['test.op2']) + tracer.trace('test.op1') do + tracer.trace('test.op2') do + assert_equal(2, ctx.length) + ctx.delete_span_if { |span| span.name == 'test.op1' } + assert_equal(1, ctx.length) + ctx.each_span do |s| + action.call_with_name(s.name) + end + assert_equal(false, ctx.finished?, 'context is not finished as op2 is not finished') + tracer.trace('test.op3') do + end + assert_equal(2, ctx.length) + ctx.delete_span_if { |span| span.name == 'test.op3' } + assert_equal(1, ctx.length) + end + assert_equal(0, ctx.length, 'op2 has been finished, so context has been finished too') + end + action.verify + end + + def test_max_length + tracer = get_test_tracer + + ctx = Datadog::Context.new + assert_equal(Datadog::Context::DEFAULT_MAX_LENGTH, ctx.max_length) + + max_length = 3 + ctx = Datadog::Context.new(max_length: max_length) + assert_equal(max_length, ctx.max_length) + + spans = [] + (max_length * 2).times do |i| + span = tracer.start_span("test.op#{i}", child_of: ctx) + spans << span + end + + assert_equal(max_length, ctx.length) + trace, = ctx.get + assert_nil(trace) + + spans.each(&:finish) + + assert_equal(0, ctx.length, "context #{ctx}") + end end class ThreadLocalContextTest < Minitest::Test diff --git a/test/span_test.rb b/test/span_test.rb index c99a2aa0470..d9626957339 100644 --- a/test/span_test.rb +++ b/test/span_test.rb @@ -1,6 +1,5 @@ require 'helper' require 'ddtrace/span' - class SpanTest < Minitest::Test def test_span_finish tracer = nil From d58cba9d4240cdf7fd2ec7c416406ac88abb3377 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 30 Mar 2018 13:52:24 -0400 Subject: [PATCH 48/72] Changed: Context flushing to be inactive by default. Provide :partial_flush or configuration options to enable. --- lib/ddtrace/tracer.rb | 22 +++++++++++++++------- test/context_flush_test.rb | 17 ++++++++++++----- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/lib/ddtrace/tracer.rb b/lib/ddtrace/tracer.rb index eab03865157..a571b2cd5b3 100644 --- a/lib/ddtrace/tracer.rb +++ b/lib/ddtrace/tracer.rb @@ -98,7 +98,7 @@ def initialize(options = {}) @provider = options.fetch(:context_provider, Datadog::DefaultContextProvider.new) @provider ||= Datadog::DefaultContextProvider.new # @provider should never be nil - @context_flush = Datadog::ContextFlush.new(options) + @context_flush = Datadog::ContextFlush.new(options) if options[:partial_flush] @mutex = Mutex.new @services = {} @@ -307,14 +307,22 @@ def record(context) context = context.context if context.is_a?(Datadog::Span) return if context.nil? trace, sampled = context.get - if sampled - if trace.nil? || trace.empty? - @context_flush.each_partial_trace(context) do |t| - write(t) + + # If context flushing is configured... + if @context_flush + if sampled + if trace.nil? || trace.empty? + @context_flush.each_partial_trace(context) do |t| + write(t) + end + else + write(trace) end - else - write(trace) end + # Default behavior + else + ready = !trace.nil? && !trace.empty? && sampled + write(trace) if ready end end diff --git a/test/context_flush_test.rb b/test/context_flush_test.rb index 947ac531796..f6ff9ae8015 100644 --- a/test/context_flush_test.rb +++ b/test/context_flush_test.rb @@ -342,14 +342,21 @@ def test_partial_caterpillar def test_tracer_configure tracer = get_test_tracer - old_context_flush = tracer.context_flush - tracer.configure() - assert_equal(old_context_flush, tracer.context_flush, 'the same context_flush should be reused') + # By default, context flush doesn't exist. + assert_nil(tracer.context_flush) + # If given a partial_flush option, then uses default context flush. + flush_tracer = Datadog::Tracer.new(writer: FauxWriter.new, partial_flush: true) + refute_nil(flush_tracer.context_flush) + + # If not configured with any flush options, context flush still doesn't exist. + tracer.configure + assert_nil(tracer.context_flush) + + # If configured with flush options, context flush gets set. tracer.configure(min_spans_before_partial_flush: 3, max_spans_before_partial_flush: 3) - - refute_equal(old_context_flush, tracer.context_flush, 'another context_flush should be have been created') + refute_nil(tracer.context_flush) end def test_tracer_hard_limit_overrides_soft_limit From d1c61e1394b854b715e2cfa5b47b0cfffd402119 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 30 Mar 2018 14:02:48 -0400 Subject: [PATCH 49/72] Added: `partial_flush` tracer option to documentation. --- docs/GettingStarted.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index a997c256f41..ebec956d338 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -798,6 +798,7 @@ Available options are: - ``env``: set the environment. Rails users may set it to ``Rails.env`` to use their application settings. - ``tags``: set global tags that should be applied to all spans. Defaults to an empty hash - ``log``: defines a custom logger. + - ``partial_flush``: set to ``true`` to enable partial trace flushing (for long running traces.) Disabled by default. *Experimental.* #### Custom logging From 603c99d5208ac62ac417c82fa9dd7ed50dc0a148 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 6 Apr 2018 12:13:14 -0400 Subject: [PATCH 50/72] Changed: Make context flush methods on context private. --- lib/ddtrace/context.rb | 61 +++++++++++++++++------------------- lib/ddtrace/context_flush.rb | 10 +++--- test/context_flush_test.rb | 10 +++--- test/context_test.rb | 38 +++++++++++----------- 4 files changed, 58 insertions(+), 61 deletions(-) diff --git a/lib/ddtrace/context.rb b/lib/ddtrace/context.rb index 8b852a6baf3..03778be3239 100644 --- a/lib/ddtrace/context.rb +++ b/lib/ddtrace/context.rb @@ -29,16 +29,6 @@ def initialize(options = {}) reset(options) end - def reset(options = {}) - @trace = [] - @parent_trace_id = options.fetch(:trace_id, nil) - @parent_span_id = options.fetch(:span_id, nil) - @sampled = options.fetch(:sampled, false) - @sampling_priority = options.fetch(:sampling_priority, nil) - @finished_spans = 0 - @current_span = nil - end - def trace_id @mutex.synchronize do @parent_trace_id @@ -73,17 +63,6 @@ def current_span end end - def set_current_span(span) - @current_span = span - if span - @parent_trace_id = span.trace_id - @parent_span_id = span.span_id - @sampled = span.sampled - else - @parent_span_id = nil - end - end - # Add a span to the context trace list, keeping it as the last active span. def add_span(span) @mutex.synchronize do @@ -124,12 +103,6 @@ def close_span(span) end end - # Returns if the trace for the current Context is finished or not. - # Low-level internal function, not thread-safe. - def check_finished_spans - @finished_spans > 0 && @trace.length == @finished_spans - end - # Returns if the trace for the current Context is finished or not. A \Context # is considered finished if all spans in this context are finished. def finished? @@ -175,6 +148,35 @@ def to_s end end + private + + def reset(options = {}) + @trace = [] + @parent_trace_id = options.fetch(:trace_id, nil) + @parent_span_id = options.fetch(:span_id, nil) + @sampled = options.fetch(:sampled, false) + @sampling_priority = options.fetch(:sampling_priority, nil) + @finished_spans = 0 + @current_span = nil + end + + def set_current_span(span) + @current_span = span + if span + @parent_trace_id = span.trace_id + @parent_span_id = span.span_id + @sampled = span.sampled + else + @parent_span_id = nil + end + end + + # Returns if the trace for the current Context is finished or not. + # Low-level internal function, not thread-safe. + def check_finished_spans + @finished_spans > 0 && @trace.length == @finished_spans + end + def attach_sampling_priority @trace.first.set_metric( Ext::DistributedTracing::SAMPLING_PRIORITY_KEY, @@ -225,11 +227,6 @@ def delete_span_if end end end - - private :reset - private :check_finished_spans - private :set_current_span - private :attach_sampling_priority end # ThreadLocalContext can be used as a tracer global reference to create diff --git a/lib/ddtrace/context_flush.rb b/lib/ddtrace/context_flush.rb index b44792d1e3d..b74d5e0d349 100644 --- a/lib/ddtrace/context_flush.rb +++ b/lib/ddtrace/context_flush.rb @@ -52,7 +52,7 @@ def partial_traces(context) # 1st step, taint all parents of an unfinished span as unflushable unflushable_ids = Set.new - context.each_span do |span| + context.send(:each_span) do |span| next if span.finished? || unflushable_ids.include?(span.span_id) unflushable_ids.add span.span_id while span.parent @@ -66,7 +66,7 @@ def partial_traces(context) # children but only for the ones we're interested it. roots = [] children_map = {} - context.each_span do |span| + context.send(:each_span) do |span| # There's no point in trying to put the real root in those partial roots, if # it's flushable, the default algorithm would figure way more quickly. if span.parent && !unflushable_ids.include?(span.span_id) @@ -102,14 +102,14 @@ def partial_flush(context) # We need to reject by span ID and not by value, because a span # value may be altered (typical example: it's finished by some other thread) # since we lock only the context, not all the spans which belong to it. - context.delete_span_if { |span| flushed_ids.include? span.span_id } + context.send(:delete_span_if) { |span| flushed_ids.include? span.span_id } traces end # Performs an operation which each partial trace it can get from the context. def each_partial_trace(context) - start_time = context.start_time - length = context.length + start_time = context.send(:start_time) + length = context.send(:length) # Stop and do not flush anything if there are not enough spans. return if length <= @min_spans_before_partial_flush # If there are enough spans, but not too many, check for start time. diff --git a/test/context_flush_test.rb b/test/context_flush_test.rb index f6ff9ae8015..c238ba0b411 100644 --- a/test/context_flush_test.rb +++ b/test/context_flush_test.rb @@ -65,7 +65,7 @@ def test_each_partial_trace_typical_not_enough_traces flunk("nothing should be partially flushed, got: #{t}") end - assert_equal(0, context.length, 'everything should be written by now') + assert_equal(0, context.send(:length), 'everything should be written by now') end def test_each_partial_trace_typical @@ -132,7 +132,7 @@ def test_each_partial_trace_typical action12.verify action3456.verify - assert_equal(0, context.length, 'everything should be written by now') + assert_equal(0, context.send(:length), 'everything should be written by now') end # rubocop:disable Metrics/MethodLength @@ -177,7 +177,7 @@ def test_each_partial_trace_mixed end end - assert_equal(8, context.length) + assert_equal(8, context.send(:length)) [root, child1, child3, child6].each do |span| span.finish @@ -253,7 +253,7 @@ def test_each_partial_trace_mixed flunk("nothing should be partially flushed, got: #{t}") end - assert_equal(0, context.length, 'everything should be written by now') + assert_equal(0, context.send(:length), 'everything should be written by now') end end @@ -322,7 +322,7 @@ def test_partial_caterpillar # - then 99 spans (10 from 1st batch, 89 from second batch) are put in context # - then the 101th comes (the 90th from the second batch) and triggers a flush of everything but root span # - then the last 10 spans from second batch are thrown in, so that's 10 left + the root span - assert_equal(1 + MIN_SPANS, tracer.call_context.length, 'some spans should have been sent') + assert_equal(1 + MIN_SPANS, tracer.call_context.send(:length), 'some spans should have been sent') assert_equal(MAX_SPANS, spans.length) spans.each do |span| write1.call_with_name(span.name) diff --git a/test/context_test.rb b/test/context_test.rb index 0d9aebcde7f..97a8f3bf2d3 100644 --- a/test/context_test.rb +++ b/test/context_test.rb @@ -278,30 +278,30 @@ def test_length tracer = get_test_tracer ctx = Datadog::Context.new - assert_equal(0, ctx.length) + assert_equal(0, ctx.send(:length)) 10.times do |i| span = Datadog::Span.new(tracer, "test.op#{i}") - assert_equal(i, ctx.length) + assert_equal(i, ctx.send(:length)) ctx.add_span(span) - assert_equal(i + 1, ctx.length) + assert_equal(i + 1, ctx.send(:length)) ctx.close_span(span) - assert_equal(i + 1, ctx.length) + assert_equal(i + 1, ctx.send(:length)) end ctx.get - assert_equal(0, ctx.length) + assert_equal(0, ctx.send(:length)) end def test_start_time tracer = get_test_tracer ctx = tracer.call_context - assert_nil(ctx.start_time) + assert_nil(ctx.send(:start_time)) tracer.trace('test.op') do |span| - assert_equal(span.start_time, ctx.start_time) + assert_equal(span.start_time, ctx.send(:start_time)) end - assert_nil(ctx.start_time) + assert_nil(ctx.send(:start_time)) end def test_each_span @@ -311,7 +311,7 @@ def test_each_span action = MiniTest::Mock.new action.expect(:call_with_name, nil, ['test.op']) - ctx.each_span do |s| + ctx.send(:each_span) do |s| action.call_with_name(s.name) end action.verify @@ -325,20 +325,20 @@ def test_delete_span_if action.expect(:call_with_name, nil, ['test.op2']) tracer.trace('test.op1') do tracer.trace('test.op2') do - assert_equal(2, ctx.length) - ctx.delete_span_if { |span| span.name == 'test.op1' } - assert_equal(1, ctx.length) - ctx.each_span do |s| + assert_equal(2, ctx.send(:length)) + ctx.send(:delete_span_if) { |span| span.name == 'test.op1' } + assert_equal(1, ctx.send(:length)) + ctx.send(:each_span) do |s| action.call_with_name(s.name) end assert_equal(false, ctx.finished?, 'context is not finished as op2 is not finished') tracer.trace('test.op3') do end - assert_equal(2, ctx.length) - ctx.delete_span_if { |span| span.name == 'test.op3' } - assert_equal(1, ctx.length) + assert_equal(2, ctx.send(:length)) + ctx.send(:delete_span_if) { |span| span.name == 'test.op3' } + assert_equal(1, ctx.send(:length)) end - assert_equal(0, ctx.length, 'op2 has been finished, so context has been finished too') + assert_equal(0, ctx.send(:length), 'op2 has been finished, so context has been finished too') end action.verify end @@ -359,13 +359,13 @@ def test_max_length spans << span end - assert_equal(max_length, ctx.length) + assert_equal(max_length, ctx.send(:length)) trace, = ctx.get assert_nil(trace) spans.each(&:finish) - assert_equal(0, ctx.length, "context #{ctx}") + assert_equal(0, ctx.send(:length), "context #{ctx}") end end From 2b37cf18f2e1773f7726eafda09a4d95be00fa76 Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 6 Apr 2018 14:36:09 -0400 Subject: [PATCH 51/72] Fixed: ActionController patching conflict with other libraries that patch #process_action --- lib/ddtrace/contrib/rails/core_extensions.rb | 99 ++++++++++++++------ 1 file changed, 69 insertions(+), 30 deletions(-) diff --git a/lib/ddtrace/contrib/rails/core_extensions.rb b/lib/ddtrace/contrib/rails/core_extensions.rb index ecd3f41cbc2..1252f99b7b7 100644 --- a/lib/ddtrace/contrib/rails/core_extensions.rb +++ b/lib/ddtrace/contrib/rails/core_extensions.rb @@ -157,40 +157,79 @@ def patch_action_controller def patch_process_action do_once(:patch_process_action) do - ::ActionController::Instrumentation.class_eval do - def process_action_with_datadog(*args) - # mutable payload with a tracing context that is used in two different - # signals; it propagates the request span so that it can be finished - # no matter what - payload = { - controller: self.class, - action: action_name, - headers: { - # The exception this controller was given in the request, - # which is typical if the controller is configured to handle exceptions. - request_exception: request.headers['action_dispatch.exception'] - }, - tracing_context: {} - } - - begin - # process and catch request exceptions - Datadog::Contrib::Rails::ActionController.start_processing(payload) - result = process_action_without_datadog(*args) - payload[:status] = response.status - result - rescue Exception => e - payload[:exception] = [e.class.name, e.message] - payload[:exception_object] = e - raise e + if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.0.0') + # Patch Rails controller base class + ::ActionController::Metal.send(:prepend, ActionControllerPatch) + else + # Rewrite module that gets composed into the Rails controller base class + ::ActionController::Instrumentation.class_eval do + def process_action_with_datadog(*args) + # mutable payload with a tracing context that is used in two different + # signals; it propagates the request span so that it can be finished + # no matter what + payload = { + controller: self.class, + action: action_name, + headers: { + # The exception this controller was given in the request, + # which is typical if the controller is configured to handle exceptions. + request_exception: request.headers['action_dispatch.exception'] + }, + tracing_context: {} + } + + begin + # process and catch request exceptions + Datadog::Contrib::Rails::ActionController.start_processing(payload) + result = process_action_without_datadog(*args) + payload[:status] = response.status + result + rescue Exception => e + payload[:exception] = [e.class.name, e.message] + payload[:exception_object] = e + raise e + end + ensure + Datadog::Contrib::Rails::ActionController.finish_processing(payload) end - ensure - Datadog::Contrib::Rails::ActionController.finish_processing(payload) + + alias_method :process_action_without_datadog, :process_action + alias_method :process_action, :process_action_with_datadog end + end + end + end - alias_method :process_action_without_datadog, :process_action - alias_method :process_action, :process_action_with_datadog + # ActionController patch for Ruby 2.0+ + module ActionControllerPatch + def process_action(*args) + # mutable payload with a tracing context that is used in two different + # signals; it propagates the request span so that it can be finished + # no matter what + payload = { + controller: self.class, + action: action_name, + headers: { + # The exception this controller was given in the request, + # which is typical if the controller is configured to handle exceptions. + request_exception: request.headers['action_dispatch.exception'] + }, + tracing_context: {} + } + + begin + # process and catch request exceptions + Datadog::Contrib::Rails::ActionController.start_processing(payload) + result = super(*args) + payload[:status] = response.status + result + rescue Exception => e + payload[:exception] = [e.class.name, e.message] + payload[:exception_object] = e + raise e end + ensure + Datadog::Contrib::Rails::ActionController.finish_processing(payload) end end end From d09bc4a90f23b9bbf8340afd62d37328f401731d Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 6 Apr 2018 17:23:40 -0400 Subject: [PATCH 52/72] Changed: Target removal version for Rack request span. --- lib/ddtrace/contrib/rack/middlewares.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ddtrace/contrib/rack/middlewares.rb b/lib/ddtrace/contrib/rack/middlewares.rb index 2404f8b7f5e..40e96a211d9 100644 --- a/lib/ddtrace/contrib/rack/middlewares.rb +++ b/lib/ddtrace/contrib/rack/middlewares.rb @@ -156,7 +156,7 @@ def get_request_id(headers, env) :datadog_rack_request_span is considered an internal symbol in the Rack env, and has been been DEPRECATED. Public support for its usage is discontinued. If you need the Rack request span, try using `Datadog.tracer.active_span`. - This key will be removed in version 0.13.0).freeze + This key will be removed in version 0.14.0).freeze def add_deprecation_warnings(env) env.instance_eval do From 936ffef34951386a579cfb95e4e3ede9ea6aa87c Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 9 Apr 2018 12:02:28 -0400 Subject: [PATCH 53/72] Fixed: Warnings in tracer. --- lib/ddtrace/tracer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ddtrace/tracer.rb b/lib/ddtrace/tracer.rb index a571b2cd5b3..6ffbfce3e1f 100644 --- a/lib/ddtrace/tracer.rb +++ b/lib/ddtrace/tracer.rb @@ -98,7 +98,7 @@ def initialize(options = {}) @provider = options.fetch(:context_provider, Datadog::DefaultContextProvider.new) @provider ||= Datadog::DefaultContextProvider.new # @provider should never be nil - @context_flush = Datadog::ContextFlush.new(options) if options[:partial_flush] + @context_flush = options[:partial_flush] ? Datadog::ContextFlush.new(options) : nil @mutex = Mutex.new @services = {} From edae86d6f98d4893e4dce196174a685f4aeee2f0 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 9 Apr 2018 12:02:46 -0400 Subject: [PATCH 54/72] Added: Log helpers for suppressing errors. --- spec/support/log_helpers.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/spec/support/log_helpers.rb b/spec/support/log_helpers.rb index 491dffaef6d..606ef449459 100644 --- a/spec/support/log_helpers.rb +++ b/spec/support/log_helpers.rb @@ -12,4 +12,14 @@ def self.without_warnings $VERBOSE = v end end + + def without_errors + level = Datadog::Tracer.log.level + Datadog::Tracer.log.level = Logger::FATAL + begin + yield + ensure + Datadog::Tracer.log.level = level + end + end end From c8d3c6d8ec7fcf88768b6d040b9281e50dafa100 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 9 Apr 2018 12:04:10 -0400 Subject: [PATCH 55/72] Fixed: Broken configuration for Racecar. --- lib/ddtrace/contrib/racecar/patcher.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ddtrace/contrib/racecar/patcher.rb b/lib/ddtrace/contrib/racecar/patcher.rb index 35e765a12b7..3443194ba90 100644 --- a/lib/ddtrace/contrib/racecar/patcher.rb +++ b/lib/ddtrace/contrib/racecar/patcher.rb @@ -76,8 +76,8 @@ def compatible? # could leak into the new trace. This "cleans" current context, # preventing such a leak. def ensure_clean_context! - return unless tracer.call_context.current_span - tracer.provider.context = Context.new + return unless configuration[:tracer].call_context.current_span + configuration[:tracer].provider.context = Context.new end end end From 67593a007d28a89a7a225410ca61c46c5c8d13ff Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 9 Apr 2018 12:05:57 -0400 Subject: [PATCH 56/72] Added: Service name default for Racecar traces. --- lib/ddtrace/contrib/racecar/patcher.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ddtrace/contrib/racecar/patcher.rb b/lib/ddtrace/contrib/racecar/patcher.rb index 3443194ba90..7bb8d7b8b89 100644 --- a/lib/ddtrace/contrib/racecar/patcher.rb +++ b/lib/ddtrace/contrib/racecar/patcher.rb @@ -17,14 +17,14 @@ module Patcher on_subscribe do # Subscribe to single messages - subscription(self::NAME_MESSAGE, {}, configuration[:tracer], &method(:process)).tap do |subscription| - subscription.before_trace(&method(:ensure_clean_context!)) + subscription(self::NAME_MESSAGE, { service: configuration[:service_name] }, configuration[:tracer], &method(:process)).tap do |subscription| + subscription.before_trace { ensure_clean_context! } subscription.subscribe('process_message.racecar') end # Subscribe to batch messages - subscription(self::NAME_BATCH, {}, configuration[:tracer], &method(:process)).tap do |subscription| - subscription.before_trace(&method(:ensure_clean_context!)) + subscription(self::NAME_BATCH, { service: configuration[:service_name] }, configuration[:tracer], &method(:process)).tap do |subscription| + subscription.before_trace { ensure_clean_context! } subscription.subscribe('process_batch.racecar') end end From b976f8f98508b7a4378c8128e2a6dbddb98e6d83 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 9 Apr 2018 12:07:19 -0400 Subject: [PATCH 57/72] Added: Error handling around subscription handlers, support for 3.x and refactors for subscription. --- .../notifications/subscription.rb | 121 ++++++++++++++---- lib/ddtrace/contrib/racecar/patcher.rb | 14 +- .../notifications/subscription_spec.rb | 40 ++++++ 3 files changed, 148 insertions(+), 27 deletions(-) diff --git a/lib/ddtrace/contrib/active_support/notifications/subscription.rb b/lib/ddtrace/contrib/active_support/notifications/subscription.rb index fde3891476d..355e9ea8307 100644 --- a/lib/ddtrace/contrib/active_support/notifications/subscription.rb +++ b/lib/ddtrace/contrib/active_support/notifications/subscription.rb @@ -7,39 +7,39 @@ class Subscription attr_reader \ :tracer, :span_name, - :options, - :block + :options def initialize(tracer, span_name, options, &block) raise ArgumentError, 'Must be given a block!' unless block_given? @tracer = tracer @span_name = span_name @options = options - @block = block - @before_trace_callbacks = [] - @after_trace_callbacks = [] + @handler = Handler.new(&block) + @callbacks = Callbacks.new end - def before_trace(&block) - @before_trace_callbacks << block if block_given? + # ActiveSupport 3.x calls this + def call(name, start, finish, id, payload) + start_span(name, id, payload, start) + finish_span(name, id, payload, finish) end - def after_trace(&block) - @after_trace_callbacks << block if block_given? + # ActiveSupport 4+ calls this on start + def start(name, id, payload) + start_span(name, id, payload) end - def start(_name, _id, _payload) - run_callbacks(@before_trace_callbacks) - tracer.trace(@span_name, @options) + # ActiveSupport 4+ calls this on finish + def finish(name, id, payload) + finish_span(name, id, payload) end - def finish(name, id, payload) - tracer.active_span.tap do |span| - return nil if span.nil? - block.call(span, name, id, payload) - span.finish - run_callbacks(@after_trace_callbacks) - end + def before_trace(&block) + callbacks.add(:before_trace, &block) if block_given? + end + + def after_trace(&block) + callbacks.add(:after_trace, &block) if block_given? end def subscribe(pattern) @@ -63,19 +63,90 @@ def unsubscribe_all protected + attr_reader \ + :handler, + :callbacks + + def start_span(name, id, payload, start = nil) + # Run callbacks + callbacks.run(name, :before_trace, id, payload, start) + + # Start a trace + tracer.trace(@span_name, @options).tap do |span| + # Assign start time if provided + span.start_time = start unless start.nil? + end + end + + def finish_span(name, id, payload, finish = nil) + tracer.active_span.tap do |span| + # If no active span, return. + return nil if span.nil? + + # Run handler for event + handler.run(span, name, id, payload) + + # Finish the span + span.finish(finish) + + # Run callbacks + callbacks.run(name, :after_trace, span, id, payload, finish) + end + end + # Pattern => ActiveSupport:Notifications::Subscribers def subscribers @subscribers ||= {} end - def run_callbacks(callbacks) - callbacks.each do |callback| - begin - callback.call - rescue StandardError => e - Datadog::Tracer.log.debug("ActiveSupport::Notifications callback failed: #{e.message}") + # Wrapper for subscription handler + class Handler + attr_reader :block + + def initialize(&block) + @block = block + end + + def run(span, name, id, payload) + run!(span, name, id, payload) + rescue StandardError => e + Datadog::Tracer.log.error("ActiveSupport::Notifications handler for '#{name}' failed: #{e.message}") + end + + def run!(*args) + @block.call(*args) + end + end + + # Wrapper for subscription callbacks + class Callbacks + attr_reader :blocks + + def initialize + @blocks = {} + end + + def add(key, &block) + blocks_for(key) << block if block_given? + end + + def run(event, key, *args) + blocks_for(key).each do |callback| + begin + callback.call(event, key, *args) + rescue StandardError => e + Datadog::Tracer.log.error( + "ActiveSupport::Notifications '#{key}' callback for '#{event}' failed: #{e.message}" + ) + end end end + + private + + def blocks_for(key) + blocks[key] ||= [] + end end end end diff --git a/lib/ddtrace/contrib/racecar/patcher.rb b/lib/ddtrace/contrib/racecar/patcher.rb index 7bb8d7b8b89..f7ff4f7fd9a 100644 --- a/lib/ddtrace/contrib/racecar/patcher.rb +++ b/lib/ddtrace/contrib/racecar/patcher.rb @@ -17,13 +17,23 @@ module Patcher on_subscribe do # Subscribe to single messages - subscription(self::NAME_MESSAGE, { service: configuration[:service_name] }, configuration[:tracer], &method(:process)).tap do |subscription| + subscription( + self::NAME_MESSAGE, + { service: configuration[:service_name] }, + configuration[:tracer], + &method(:process) + ).tap do |subscription| subscription.before_trace { ensure_clean_context! } subscription.subscribe('process_message.racecar') end # Subscribe to batch messages - subscription(self::NAME_BATCH, { service: configuration[:service_name] }, configuration[:tracer], &method(:process)).tap do |subscription| + subscription( + self::NAME_BATCH, + { service: configuration[:service_name] }, + configuration[:tracer], + &method(:process) + ).tap do |subscription| subscription.before_trace { ensure_clean_context! } subscription.subscribe('process_batch.racecar') end diff --git a/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb b/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb index d6cc97ff086..f19855f6379 100644 --- a/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb +++ b/spec/ddtrace/contrib/active_support/notifications/subscription_spec.rb @@ -18,6 +18,44 @@ let(:spy) { double('spy') } describe 'behavior' do + describe '#call' do + subject(:result) { subscription.call(name, start, finish, id, payload) } + let(:name) { double('name') } + let(:start) { double('start') } + let(:finish) { double('finish') } + let(:id) { double('id') } + let(:payload) { double('payload') } + + let(:span) { instance_double(Datadog::Span) } + + it do + expect(tracer).to receive(:trace).with(span_name, options).and_return(span).ordered + expect(span).to receive(:start_time=).with(start).and_return(span).ordered + expect(tracer).to receive(:active_span).and_return(span).ordered + expect(spy).to receive(:call).with(span, name, id, payload).ordered + expect(span).to receive(:finish).with(finish).and_return(span).ordered + is_expected.to be(span) + end + + context 'when block raises an error' do + let(:block) do + Proc.new do |span, name, id, payload| + raise ArgumentError.new('Fail!') + end + end + + around(:each) { |example| without_errors { example.run } } + + it 'finishes tracing anyways' do + expect(tracer).to receive(:trace).with(span_name, options).and_return(span).ordered + expect(span).to receive(:start_time=).with(start).and_return(span).ordered + expect(tracer).to receive(:active_span).and_return(span).ordered + expect(span).to receive(:finish).with(finish).and_return(span).ordered + is_expected.to be(span) + end + end + end + describe '#start' do subject(:result) { subscription.start(name, id, payload) } let(:name) { double('name') } @@ -71,6 +109,7 @@ context 'that raises an error' do let(:callback_block) { Proc.new { callback_spy.call; raise ArgumentError.new('Fail!') } } + around(:each) { |example| without_errors { example.run } } it_behaves_like 'a before_trace callback' end end @@ -103,6 +142,7 @@ context 'that raises an error' do let(:callback_block) { Proc.new { callback_spy.call; raise ArgumentError.new('Fail!') } } + around(:each) { |example| without_errors { example.run } } it_behaves_like 'an after_trace callback' end end From d9c1f83d754ee573b7425d503e086eb9834a6138 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 9 Apr 2018 13:07:19 -0400 Subject: [PATCH 58/72] Changed: Log subscription errors as debug (so as not to spam logs.) --- .../contrib/active_support/notifications/subscription.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ddtrace/contrib/active_support/notifications/subscription.rb b/lib/ddtrace/contrib/active_support/notifications/subscription.rb index 355e9ea8307..f500cf413ae 100644 --- a/lib/ddtrace/contrib/active_support/notifications/subscription.rb +++ b/lib/ddtrace/contrib/active_support/notifications/subscription.rb @@ -110,7 +110,7 @@ def initialize(&block) def run(span, name, id, payload) run!(span, name, id, payload) rescue StandardError => e - Datadog::Tracer.log.error("ActiveSupport::Notifications handler for '#{name}' failed: #{e.message}") + Datadog::Tracer.log.debug("ActiveSupport::Notifications handler for '#{name}' failed: #{e.message}") end def run!(*args) @@ -135,7 +135,7 @@ def run(event, key, *args) begin callback.call(event, key, *args) rescue StandardError => e - Datadog::Tracer.log.error( + Datadog::Tracer.log.debug( "ActiveSupport::Notifications '#{key}' callback for '#{event}' failed: #{e.message}" ) end From 6a1d5c05433f6209edbc2b0ffabc47ec2ca801eb Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 10 Apr 2018 13:24:48 -0400 Subject: [PATCH 59/72] Fixed: Wrong value for #min_spans_before_partial_flush --- lib/ddtrace/tracer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ddtrace/tracer.rb b/lib/ddtrace/tracer.rb index 6ffbfce3e1f..7afb7d74d82 100644 --- a/lib/ddtrace/tracer.rb +++ b/lib/ddtrace/tracer.rb @@ -125,7 +125,7 @@ def configure(options = {}) sampler = options.fetch(:sampler, nil) priority_sampling = options[:priority_sampling] max_spans_before_partial_flush = options.fetch(:max_spans_before_partial_flush, nil) - min_spans_before_partial_flush = options.fetch(:max_spans_before_partial_flush, nil) + min_spans_before_partial_flush = options.fetch(:min_spans_before_partial_flush, nil) partial_flush_timeout = options.fetch(:partial_flush_timeout, nil) @enabled = enabled unless enabled.nil? From 9ebff740805040fd04814d713f463a117b7d28de Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 6 Apr 2018 15:59:06 -0400 Subject: [PATCH 60/72] Refactored: ActiveRecord Utils functions from Rails to ActiveRecord. --- lib/ddtrace/contrib/active_record/patcher.rb | 6 +- lib/ddtrace/contrib/active_record/utils.rb | 85 +++++++++++++++++++ lib/ddtrace/contrib/rails/active_record.rb | 4 +- lib/ddtrace/contrib/rails/framework.rb | 3 +- lib/ddtrace/contrib/rails/utils.rb | 76 ----------------- .../contrib/active_record/tracer_spec.rb | 20 +++-- .../contrib/active_record/utils_spec.rb | 53 ++++++++++++ spec/support/rails_active_record_helpers.rb | 8 +- test/contrib/rails/utils_test.rb | 32 ------- .../sinatra/tracer_activerecord_test.rb | 8 +- test/helper.rb | 8 +- 11 files changed, 172 insertions(+), 131 deletions(-) create mode 100644 lib/ddtrace/contrib/active_record/utils.rb create mode 100644 spec/ddtrace/contrib/active_record/utils_spec.rb diff --git a/lib/ddtrace/contrib/active_record/patcher.rb b/lib/ddtrace/contrib/active_record/patcher.rb index d4811c6589c..69ce24d391c 100644 --- a/lib/ddtrace/contrib/active_record/patcher.rb +++ b/lib/ddtrace/contrib/active_record/patcher.rb @@ -1,6 +1,6 @@ require 'ddtrace/ext/sql' require 'ddtrace/ext/app_types' -require 'ddtrace/contrib/rails/utils' +require 'ddtrace/contrib/active_record/utils' module Datadog module Contrib @@ -10,7 +10,7 @@ module Patcher include Base register_as :active_record, auto_patch: false option :service_name, depends_on: [:tracer] do |value| - (value || Datadog::Contrib::Rails::Utils.adapter_name).tap do |v| + (value || Utils.adapter_name).tap do |v| get_option(:tracer).set_service_info(v, 'active_record', Ext::AppTypes::DB) end end @@ -59,7 +59,7 @@ def instantiation_tracing_supported? end def self.sql(_name, start, finish, _id, payload) - connection_config = Datadog::Contrib::Rails::Utils.connection_config(payload[:connection_id]) + connection_config = Utils.connection_config(payload[:connection_id]) span = get_option(:tracer).trace( "#{connection_config[:adapter_name]}.query", diff --git a/lib/ddtrace/contrib/active_record/utils.rb b/lib/ddtrace/contrib/active_record/utils.rb new file mode 100644 index 00000000000..71d77b96a04 --- /dev/null +++ b/lib/ddtrace/contrib/active_record/utils.rb @@ -0,0 +1,85 @@ +module Datadog + module Contrib + module ActiveRecord + # Common utilities for Rails + module Utils + # Return a canonical name for a type of database + def self.normalize_vendor(vendor) + case vendor + when nil + 'defaultdb' + when 'mysql2' + 'mysql' + when 'postgresql' + 'postgres' + when 'sqlite3' + 'sqlite' + else + vendor + end + end + + def self.adapter_name + connection_config[:adapter_name] + end + + def self.database_name + connection_config[:database_name] + end + + def self.adapter_host + connection_config[:adapter_host] + end + + def self.adapter_port + connection_config[:adapter_port] + end + + def self.connection_config(object_id = nil) + config = object_id.nil? ? default_connection_config : connection_config_by_id(object_id) + { + adapter_name: normalize_vendor(config[:adapter]), + adapter_host: config[:host], + adapter_port: config[:port], + database_name: config[:database] + } + end + + # Attempt to retrieve the connection from an object ID. + def self.connection_by_id(object_id) + return nil if object_id.nil? + ObjectSpace._id2ref(object_id) + rescue StandardError + nil + end + + # Attempt to retrieve the connection config from an object ID. + # Typical of ActiveSupport::Notifications `sql.active_record` + def self.connection_config_by_id(object_id) + connection = connection_by_id(object_id) + return {} if connection.nil? + + if connection.instance_variable_defined?(:@config) + connection.instance_variable_get(:@config) + else + {} + end + end + + def self.default_connection_config + return @default_connection_config unless !instance_variable_defined?(:@default_connection_config) + current_connection_name = if ::ActiveRecord::Base.respond_to?(:connection_specification_name) + ::ActiveRecord::Base.connection_specification_name + else + ::ActiveRecord::Base + end + + connection_pool = ::ActiveRecord::Base.connection_handler.retrieve_connection_pool(current_connection_name) + connection_pool.nil? ? {} : (@default_connection_config = connection_pool.spec.config) + rescue StandardError + {} + end + end + end + end +end diff --git a/lib/ddtrace/contrib/rails/active_record.rb b/lib/ddtrace/contrib/rails/active_record.rb index 1dd62a5412c..c3d80a0f63a 100644 --- a/lib/ddtrace/contrib/rails/active_record.rb +++ b/lib/ddtrace/contrib/rails/active_record.rb @@ -1,5 +1,5 @@ require 'ddtrace/ext/sql' -require 'ddtrace/contrib/rails/utils' +require 'ddtrace/contrib/active_record/utils' module Datadog module Contrib @@ -30,7 +30,7 @@ def self.instrument def self.sql(_name, start, finish, _id, payload) tracer = Datadog.configuration[:rails][:tracer] database_service = Datadog.configuration[:rails][:database_service] - connection_config = Datadog::Contrib::Rails::Utils.connection_config(payload[:connection_id]) + connection_config = Datadog::Contrib::ActiveRecord::Utils.connection_config(payload[:connection_id]) span = tracer.trace( "#{connection_config[:adapter_name]}.query", diff --git a/lib/ddtrace/contrib/rails/framework.rb b/lib/ddtrace/contrib/rails/framework.rb index 413093923de..fa62c18446b 100644 --- a/lib/ddtrace/contrib/rails/framework.rb +++ b/lib/ddtrace/contrib/rails/framework.rb @@ -1,6 +1,7 @@ require 'ddtrace/pin' require 'ddtrace/ext/app_types' +require 'ddtrace/contrib/active_record/utils' require 'ddtrace/contrib/grape/endpoint' require 'ddtrace/contrib/rack/middlewares' @@ -50,7 +51,7 @@ def self.set_database_service return unless defined?(::ActiveRecord) config = Datadog.configuration[:rails] - adapter_name = Utils.adapter_name + adapter_name = Contrib::ActiveRecord::Utils.adapter_name config[:database_service] ||= "#{config[:service_name]}-#{adapter_name}" config[:tracer].set_service_info(config[:database_service], adapter_name, Ext::AppTypes::DB) rescue => e diff --git a/lib/ddtrace/contrib/rails/utils.rb b/lib/ddtrace/contrib/rails/utils.rb index 6b861f8a56a..5bfda5bbcb3 100644 --- a/lib/ddtrace/contrib/rails/utils.rb +++ b/lib/ddtrace/contrib/rails/utils.rb @@ -23,21 +23,6 @@ def self.normalize_template_name(name) return name.to_s end - # TODO: Consider moving this out of Rails. - # Return a canonical name for a type of database - def self.normalize_vendor(vendor) - case vendor - when nil - 'defaultdb' - when 'sqlite3' - 'sqlite' - when 'postgresql' - 'postgres' - else - vendor - end - end - def self.app_name if ::Rails::VERSION::MAJOR >= 4 ::Rails.application.class.parent_name.underscore @@ -46,67 +31,6 @@ def self.app_name end end - def self.adapter_name - connection_config[:adapter_name] - end - - def self.database_name - connection_config[:database_name] - end - - def self.adapter_host - connection_config[:adapter_host] - end - - def self.adapter_port - connection_config[:adapter_port] - end - - def self.connection_config(object_id = nil) - config = object_id.nil? ? default_connection_config : connection_config_by_id(object_id) - { - adapter_name: normalize_vendor(config[:adapter]), - adapter_host: config[:host], - adapter_port: config[:port], - database_name: config[:database] - } - end - - # Attempt to retrieve the connection from an object ID. - def self.connection_by_id(object_id) - return nil if object_id.nil? - ObjectSpace._id2ref(object_id) - rescue StandardError - nil - end - - # Attempt to retrieve the connection config from an object ID. - # Typical of ActiveSupport::Notifications `sql.active_record` - def self.connection_config_by_id(object_id) - connection = connection_by_id(object_id) - return {} if connection.nil? - - if connection.instance_variable_defined?(:@config) - connection.instance_variable_get(:@config) - else - {} - end - end - - def self.default_connection_config - return @default_connection_config unless @default_connection_config.nil? - current_connection_name = if ::ActiveRecord::Base.respond_to?(:connection_specification_name) - ::ActiveRecord::Base.connection_specification_name - else - ::ActiveRecord::Base - end - - connection_pool = ::ActiveRecord::Base.connection_handler.retrieve_connection_pool(current_connection_name) - connection_pool.nil? ? {} : (@default_connection_config = connection_pool.spec.config) - rescue StandardError - {} - end - def self.exception_is_error?(exception) if defined?(::ActionDispatch::ExceptionWrapper) # Gets the equivalent status code for the exception (not all are 5XX) diff --git a/spec/ddtrace/contrib/active_record/tracer_spec.rb b/spec/ddtrace/contrib/active_record/tracer_spec.rb index 3a92fad4273..ff186063e66 100644 --- a/spec/ddtrace/contrib/active_record/tracer_spec.rb +++ b/spec/ddtrace/contrib/active_record/tracer_spec.rb @@ -8,11 +8,21 @@ let(:configuration_options) { { tracer: tracer } } before(:each) do + # Prevent extra spans during tests + Article.count + + # Reset options (that might linger from other tests) + Datadog.configuration[:active_record].reset_options! + Datadog.configure do |c| c.use :active_record, configuration_options end end + after(:each) do + Datadog.configuration[:active_record].reset_options! + end + it 'calls the instrumentation when is used standalone' do Article.count spans = tracer.writer.spans @@ -20,14 +30,14 @@ # expect service and trace is sent expect(spans.size).to eq(1) - expect(services['mysql2']).to eq({'app'=>'active_record', 'app_type'=>'db'}) + expect(services['mysql']).to eq({'app'=>'active_record', 'app_type'=>'db'}) span = spans[0] - expect(span.service).to eq('mysql2') - expect(span.name).to eq('mysql2.query') + expect(span.service).to eq('mysql') + expect(span.name).to eq('mysql.query') expect(span.span_type).to eq('sql') expect(span.resource.strip).to eq('SELECT COUNT(*) FROM `articles`') - expect(span.get_tag('active_record.db.vendor')).to eq('mysql2') + expect(span.get_tag('active_record.db.vendor')).to eq('mysql') expect(span.get_tag('active_record.db.name')).to eq('mysql') expect(span.get_tag('active_record.db.cached')).to eq(nil) expect(span.get_tag('out.host')).to eq('127.0.0.1') @@ -45,7 +55,7 @@ context 'is not set' do let(:configuration_options) { super().merge({ service_name: nil }) } - it { expect(query_span.service).to eq('mysql2') } + it { expect(query_span.service).to eq('mysql') } end context 'is set' do diff --git a/spec/ddtrace/contrib/active_record/utils_spec.rb b/spec/ddtrace/contrib/active_record/utils_spec.rb new file mode 100644 index 00000000000..dd656b83510 --- /dev/null +++ b/spec/ddtrace/contrib/active_record/utils_spec.rb @@ -0,0 +1,53 @@ +require 'spec_helper' + +require 'ddtrace/contrib/active_record/utils' + +RSpec.describe Datadog::Contrib::ActiveRecord::Utils do + + describe '#normalize_vendor' do + subject(:result) { described_class.normalize_vendor(value) } + + context 'when given' do + context 'nil' do + let(:value) { nil } + it { is_expected.to eq('defaultdb') } + end + + context 'sqlite3' do + let(:value) { 'sqlite3' } + it { is_expected.to eq('sqlite') } + end + + context 'mysql2' do + let(:value) { 'mysql2' } + it { is_expected.to eq('mysql') } + end + + context 'postgresql' do + let(:value) { 'postgresql' } + it { is_expected.to eq('postgres') } + end + + context 'customdb' do + let(:value) { 'customdb' } + it { is_expected.to eq(value) } + end + end + end + + describe 'regression: retrieving database without an active connection does not raise an error' do + before(:each) do + ActiveRecord::Base.establish_connection('mysql2://root:root@127.0.0.1:53306/mysql') + ActiveRecord::Base.remove_connection + end + + after(:each) { ActiveRecord::Base.establish_connection('mysql2://root:root@127.0.0.1:53306/mysql') } + + it do + expect { described_class.adapter_name }.to_not raise_error + expect { described_class.adapter_host }.to_not raise_error + expect { described_class.adapter_port }.to_not raise_error + expect { described_class.database_name }.to_not raise_error + end + end +end diff --git a/spec/support/rails_active_record_helpers.rb b/spec/support/rails_active_record_helpers.rb index 66792022036..ef8a7e75452 100644 --- a/spec/support/rails_active_record_helpers.rb +++ b/spec/support/rails_active_record_helpers.rb @@ -1,17 +1,17 @@ module RailsActiveRecordHelpers def get_adapter_name - Datadog::Contrib::Rails::Utils.adapter_name + Datadog::Contrib::ActiveRecord::Utils.adapter_name end def get_database_name - Datadog::Contrib::Rails::Utils.database_name + Datadog::Contrib::ActiveRecord::Utils.database_name end def get_adapter_host - Datadog::Contrib::Rails::Utils.adapter_host + Datadog::Contrib::ActiveRecord::Utils.adapter_host end def get_adapter_port - Datadog::Contrib::Rails::Utils.adapter_port + Datadog::Contrib::ActiveRecord::Utils.adapter_port end end diff --git a/test/contrib/rails/utils_test.rb b/test/contrib/rails/utils_test.rb index ac263fc86e2..43f56c45f85 100644 --- a/test/contrib/rails/utils_test.rb +++ b/test/contrib/rails/utils_test.rb @@ -45,36 +45,4 @@ class UtilsTest < ActiveSupport::TestCase template_name = Datadog::Contrib::Rails::Utils.normalize_template_name({}) assert_equal(template_name, '{}') end - - test 'normalize adapter name for a not defined vendor' do - vendor = Datadog::Contrib::Rails::Utils.normalize_vendor(nil) - assert_equal(vendor, 'defaultdb') - end - - test 'normalize adapter name for sqlite3' do - vendor = Datadog::Contrib::Rails::Utils.normalize_vendor('sqlite3') - assert_equal(vendor, 'sqlite') - end - - test 'normalize adapter name for postgresql' do - vendor = Datadog::Contrib::Rails::Utils.normalize_vendor('postgresql') - assert_equal(vendor, 'postgres') - end - - test 'normalize adapter name for an unknown vendor' do - vendor = Datadog::Contrib::Rails::Utils.normalize_vendor('customdb') - assert_equal(vendor, 'customdb') - end - - test 'regression: database info is available even without an active connection' do - begin - ActiveRecord::Base.remove_connection - refute_nil(Datadog::Contrib::Rails::Utils.adapter_name) - refute_nil(Datadog::Contrib::Rails::Utils.adapter_host) - refute_nil(Datadog::Contrib::Rails::Utils.adapter_port) - refute_nil(Datadog::Contrib::Rails::Utils.database_name) - ensure - ActiveRecord::Base.establish_connection - end - end end diff --git a/test/contrib/sinatra/tracer_activerecord_test.rb b/test/contrib/sinatra/tracer_activerecord_test.rb index 79db4daad96..b912d15e03c 100644 --- a/test/contrib/sinatra/tracer_activerecord_test.rb +++ b/test/contrib/sinatra/tracer_activerecord_test.rb @@ -74,10 +74,10 @@ def test_request sinatra_span = spans[0] sqlite_span = spans[spans.length - 1] - adapter_name = Datadog::Contrib::Rails::Utils.adapter_name - database_name = Datadog::Contrib::Rails::Utils.database_name - adapter_host = Datadog::Contrib::Rails::Utils.adapter_host - adapter_port = Datadog::Contrib::Rails::Utils.adapter_port + adapter_name = Datadog::Contrib::ActiveRecord::Utils.adapter_name + database_name = Datadog::Contrib::ActiveRecord::Utils.database_name + adapter_host = Datadog::Contrib::ActiveRecord::Utils.adapter_host + adapter_port = Datadog::Contrib::ActiveRecord::Utils.adapter_port assert_equal('sqlite', sqlite_span.service) assert_equal('SELECT 42', sqlite_span.resource) diff --git a/test/helper.rb b/test/helper.rb index b93cf788311..6826cdc170f 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -66,19 +66,19 @@ def get_test_services end def get_adapter_name - Datadog::Contrib::Rails::Utils.adapter_name + Datadog::Contrib::ActiveRecord::Utils.adapter_name end def get_database_name - Datadog::Contrib::Rails::Utils.database_name + Datadog::Contrib::ActiveRecord::Utils.database_name end def get_adapter_host - Datadog::Contrib::Rails::Utils.adapter_host + Datadog::Contrib::ActiveRecord::Utils.adapter_host end def get_adapter_port - Datadog::Contrib::Rails::Utils.adapter_port + Datadog::Contrib::ActiveRecord::Utils.adapter_port end # FauxWriter is a dummy writer that buffers spans locally. From f6908b94edd293dcfb27875ad13633dfb2fed214 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 9 Apr 2018 14:23:09 -0400 Subject: [PATCH 61/72] Removed: Rails::ActiveRecord in favor of standalone ActiveRecord. --- lib/ddtrace/contrib/active_record/utils.rb | 2 +- lib/ddtrace/contrib/rails/active_record.rb | 82 ---------------------- lib/ddtrace/contrib/rails/framework.rb | 38 +++++----- lib/ddtrace/contrib/rails/patcher.rb | 16 +++-- lib/ddtrace/contrib/rails/railtie.rb | 1 - test/contrib/rails/controller_test.rb | 2 +- test/contrib/rails/database_test.rb | 59 ++++++++++++---- test/contrib/rails/rack_middleware_test.rb | 8 +-- test/contrib/rails/tracer_test.rb | 11 ++- 9 files changed, 85 insertions(+), 134 deletions(-) delete mode 100644 lib/ddtrace/contrib/rails/active_record.rb diff --git a/lib/ddtrace/contrib/active_record/utils.rb b/lib/ddtrace/contrib/active_record/utils.rb index 71d77b96a04..f65731d18a8 100644 --- a/lib/ddtrace/contrib/active_record/utils.rb +++ b/lib/ddtrace/contrib/active_record/utils.rb @@ -67,7 +67,7 @@ def self.connection_config_by_id(object_id) end def self.default_connection_config - return @default_connection_config unless !instance_variable_defined?(:@default_connection_config) + return @default_connection_config if instance_variable_defined?(:@default_connection_config) current_connection_name = if ::ActiveRecord::Base.respond_to?(:connection_specification_name) ::ActiveRecord::Base.connection_specification_name else diff --git a/lib/ddtrace/contrib/rails/active_record.rb b/lib/ddtrace/contrib/rails/active_record.rb deleted file mode 100644 index c3d80a0f63a..00000000000 --- a/lib/ddtrace/contrib/rails/active_record.rb +++ /dev/null @@ -1,82 +0,0 @@ -require 'ddtrace/ext/sql' -require 'ddtrace/contrib/active_record/utils' - -module Datadog - module Contrib - module Rails - # Code used to create and handle 'mysql.query', 'postgres.query', ... spans. - module ActiveRecord - include Datadog::Patcher - - def self.instrument - # ActiveRecord is instrumented only if it's available - return unless defined?(::ActiveRecord) - - do_once(:instrument) do - # subscribe when the active record query has been processed - ::ActiveSupport::Notifications.subscribe('sql.active_record') do |*args| - sql(*args) - end - end - - if Datadog::Contrib::Rails::Patcher.active_record_instantiation_tracing_supported? - # subscribe when the active record instantiates objects - ::ActiveSupport::Notifications.subscribe('instantiation.active_record') do |*args| - instantiation(*args) - end - end - end - - def self.sql(_name, start, finish, _id, payload) - tracer = Datadog.configuration[:rails][:tracer] - database_service = Datadog.configuration[:rails][:database_service] - connection_config = Datadog::Contrib::ActiveRecord::Utils.connection_config(payload[:connection_id]) - - span = tracer.trace( - "#{connection_config[:adapter_name]}.query", - resource: payload.fetch(:sql), - service: database_service, - span_type: Datadog::Ext::SQL::TYPE - ) - - # Find out if the SQL query has been cached in this request. This meta is really - # helpful to users because some spans may have 0ns of duration because the query - # is simply cached from memory, so the notification is fired with start == finish. - cached = payload[:cached] || (payload[:name] == 'CACHE') - - # the span should have the query ONLY in the Resource attribute, - # so that the ``sql.query`` tag will be set in the agent with an - # obfuscated version - span.span_type = Datadog::Ext::SQL::TYPE - span.set_tag('rails.db.vendor', connection_config[:adapter_name]) - span.set_tag('rails.db.name', connection_config[:database_name]) - span.set_tag('rails.db.cached', cached) if cached - span.set_tag('out.host', connection_config[:adapter_host]) - span.set_tag('out.port', connection_config[:adapter_port]) - span.start_time = start - span.finish(finish) - rescue StandardError => e - Datadog::Tracer.log.debug(e.message) - end - - def self.instantiation(_name, start, finish, _id, payload) - tracer = Datadog.configuration[:rails][:tracer] - - span = tracer.trace( - 'active_record.instantiation', - resource: payload.fetch(:class_name), - span_type: 'custom' - ) - - span.service = span.parent ? span.parent.service : Datadog.configuration[:rails][:service_name] - span.set_tag('active_record.instantiation.class_name', payload.fetch(:class_name)) - span.set_tag('active_record.instantiation.record_count', payload.fetch(:record_count)) - span.start_time = start - span.finish(finish) - rescue StandardError => e - Datadog::Tracer.log.debug(e.message) - end - end - end - end -end diff --git a/lib/ddtrace/contrib/rails/framework.rb b/lib/ddtrace/contrib/rails/framework.rb index fa62c18446b..95568efdcb9 100644 --- a/lib/ddtrace/contrib/rails/framework.rb +++ b/lib/ddtrace/contrib/rails/framework.rb @@ -1,14 +1,13 @@ require 'ddtrace/pin' require 'ddtrace/ext/app_types' -require 'ddtrace/contrib/active_record/utils' +require 'ddtrace/contrib/active_record/patcher' require 'ddtrace/contrib/grape/endpoint' require 'ddtrace/contrib/rack/middlewares' require 'ddtrace/contrib/rails/core_extensions' require 'ddtrace/contrib/rails/action_controller' require 'ddtrace/contrib/rails/action_view' -require 'ddtrace/contrib/rails/active_record' require 'ddtrace/contrib/rails/active_support' require 'ddtrace/contrib/rails/utils' @@ -26,36 +25,37 @@ def self.setup config[:service_name] ||= Utils.app_name tracer = config[:tracer] - Datadog.configuration.use( - :rack, - tracer: tracer, - application: ::Rails.application, - service_name: config[:service_name], - middleware_names: config[:middleware_names], - distributed_tracing: config[:distributed_tracing] - ) + activate_rack!(config) + activate_active_record!(config) config[:controller_service] ||= config[:service_name] config[:cache_service] ||= "#{config[:service_name]}-cache" tracer.set_service_info(config[:controller_service], 'rails', Ext::AppTypes::WEB) tracer.set_service_info(config[:cache_service], 'rails', Ext::AppTypes::CACHE) - set_database_service # By default, default service would be guessed from the script # being executed, but here we know better, get it from Rails config. tracer.default_service = config[:service_name] end - def self.set_database_service - return unless defined?(::ActiveRecord) + def self.activate_rack!(config) + Datadog.configuration.use( + :rack, + tracer: config[:tracer], + application: ::Rails.application, + service_name: config[:service_name], + middleware_names: config[:middleware_names], + distributed_tracing: config[:distributed_tracing] + ) + end - config = Datadog.configuration[:rails] - adapter_name = Contrib::ActiveRecord::Utils.adapter_name - config[:database_service] ||= "#{config[:service_name]}-#{adapter_name}" - config[:tracer].set_service_info(config[:database_service], adapter_name, Ext::AppTypes::DB) - rescue => e - Tracer.log.warn("Unable to get database config (#{e}), skipping ActiveRecord instrumentation") + def self.activate_active_record!(config) + Datadog.configuration.use( + :active_record, + service_name: config[:database_service], + tracer: config[:tracer] + ) end end end diff --git a/lib/ddtrace/contrib/rails/patcher.rb b/lib/ddtrace/contrib/rails/patcher.rb index f698376f21d..7f2ce690780 100644 --- a/lib/ddtrace/contrib/rails/patcher.rb +++ b/lib/ddtrace/contrib/rails/patcher.rb @@ -6,10 +6,17 @@ module Patcher include Base register_as :rails, auto_patch: true - option :service_name + option :service_name do |value| + value || Utils.app_name + end option :controller_service option :cache_service - option :database_service + option :database_service, depends_on: [:service_name] do |value| + (value || "#{get_option(:service_name)}-#{Contrib::ActiveRecord::Utils.adapter_name}").tap do |v| + # Update ActiveRecord service name too + Datadog.configuration[:active_record][:service_name] = v + end + end option :middleware_names, default: false option :distributed_tracing, default: false option :template_base_path, default: 'views/' @@ -37,11 +44,6 @@ def compatible? defined?(::Rails::VERSION) && ::Rails::VERSION::MAJOR.to_i >= 3 end - - def active_record_instantiation_tracing_supported? - Gem.loaded_specs['activerecord'] \ - && Gem.loaded_specs['activerecord'].version >= Gem::Version.new('4.2') - end end end end diff --git a/lib/ddtrace/contrib/rails/railtie.rb b/lib/ddtrace/contrib/rails/railtie.rb index a943815632f..42345b1583a 100644 --- a/lib/ddtrace/contrib/rails/railtie.rb +++ b/lib/ddtrace/contrib/rails/railtie.rb @@ -15,7 +15,6 @@ class Railtie < Rails::Railtie Datadog::Contrib::Rails::Framework.setup Datadog::Contrib::Rails::ActionController.instrument Datadog::Contrib::Rails::ActionView.instrument - Datadog::Contrib::Rails::ActiveRecord.instrument Datadog::Contrib::Rails::ActiveSupport.instrument end end diff --git a/test/contrib/rails/controller_test.rb b/test/contrib/rails/controller_test.rb index 6bc61aa3aba..5146f59115d 100644 --- a/test/contrib/rails/controller_test.rb +++ b/test/contrib/rails/controller_test.rb @@ -117,7 +117,7 @@ class TracingControllerTest < ActionController::TestCase spans = @tracer.writer.spans # rubocop:disable Style/IdenticalConditionalBranches - if Datadog::Contrib::Rails::Patcher.active_record_instantiation_tracing_supported? + if Datadog::Contrib::ActiveRecord::Patcher.instantiation_tracing_supported? assert_equal(spans.length, 5) span_instantiation, span_database, span_request, span_cache, span_template = spans diff --git a/test/contrib/rails/database_test.rb b/test/contrib/rails/database_test.rb index fb84318283c..07934e07059 100644 --- a/test/contrib/rails/database_test.rb +++ b/test/contrib/rails/database_test.rb @@ -5,19 +5,25 @@ class DatabaseTracingTest < ActiveSupport::TestCase setup do @original_tracer = Datadog.configuration[:rails][:tracer] @tracer = get_test_tracer - Datadog.configuration[:rails][:database_service] = get_adapter_name - Datadog.configuration[:rails][:tracer] = @tracer + + Datadog.configure do |c| + c.use :rails, database_service: get_adapter_name, tracer: @tracer + end + + Datadog.configuration[:active_record][:service_name] = get_adapter_name + Datadog.configuration[:active_record][:tracer] = @tracer end teardown do Datadog.configuration[:rails][:tracer] = @original_tracer + Datadog.configuration[:active_record][:tracer] = @original_tracer end test 'active record is properly traced' do # make the query and assert the proper spans Article.count spans = @tracer.writer.spans - assert_equal(spans.length, 1) + assert_equal(1, spans.length) span = spans.first adapter_name = get_adapter_name @@ -27,9 +33,9 @@ class DatabaseTracingTest < ActiveSupport::TestCase assert_equal(span.name, "#{adapter_name}.query") assert_equal(span.span_type, 'sql') assert_equal(span.service, adapter_name) - assert_equal(span.get_tag('rails.db.vendor'), adapter_name) - assert_equal(span.get_tag('rails.db.name'), database_name) - assert_nil(span.get_tag('rails.db.cached')) + assert_equal(span.get_tag('active_record.db.vendor'), adapter_name) + assert_equal(span.get_tag('active_record.db.name'), database_name) + assert_nil(span.get_tag('active_record.db.cached')) assert_equal(adapter_host.to_s, span.get_tag('out.host')) assert_equal(adapter_port.to_s, span.get_tag('out.port')) assert_includes(span.resource, 'SELECT COUNT(*) FROM') @@ -38,7 +44,7 @@ class DatabaseTracingTest < ActiveSupport::TestCase end test 'active record traces instantiation' do - if Datadog::Contrib::Rails::Patcher.active_record_instantiation_tracing_supported? + if Datadog::Contrib::ActiveRecord::Patcher.instantiation_tracing_supported? begin Article.create(title: 'Instantiation test') @tracer.writer.spans # Clear spans @@ -51,7 +57,34 @@ class DatabaseTracingTest < ActiveSupport::TestCase instantiation_span = spans.first assert_equal(instantiation_span.name, 'active_record.instantiation') assert_equal(instantiation_span.span_type, 'custom') - assert_equal(instantiation_span.service, Datadog.configuration[:rails][:service_name]) + # Because no parent, and doesn't belong to database service + assert_equal(instantiation_span.service, 'active_record') + assert_equal(instantiation_span.resource, 'Article') + assert_equal(instantiation_span.get_tag('active_record.instantiation.class_name'), 'Article') + assert_equal(instantiation_span.get_tag('active_record.instantiation.record_count'), '1') + ensure + Article.delete_all + end + end + end + + test 'active record traces instantiation inside parent trace' do + if Datadog::Contrib::ActiveRecord::Patcher.instantiation_tracing_supported? + begin + Article.create(title: 'Instantiation test') + @tracer.writer.spans # Clear spans + + # make the query and assert the proper spans + @tracer.trace('parent.span', service: 'parent-service') do + Article.all.entries + end + spans = @tracer.writer.spans + assert_equal(3, spans.length) + instantiation_span, parent_span, _query_span = spans + + assert_equal(instantiation_span.name, 'active_record.instantiation') + assert_equal(instantiation_span.span_type, 'custom') + assert_equal(instantiation_span.service, parent_span.service) # Because within parent assert_equal(instantiation_span.resource, 'Article') assert_equal(instantiation_span.get_tag('active_record.instantiation.class_name'), 'Article') assert_equal(instantiation_span.get_tag('active_record.instantiation.record_count'), '1') @@ -70,13 +103,13 @@ class DatabaseTracingTest < ActiveSupport::TestCase # Assert correct number of spans spans = @tracer.writer.spans - assert_equal(spans.length, 2) + assert_equal(2, spans.length) # Assert cached flag not present on first query - assert_nil(spans.first.get_tag('rails.db.cached')) + assert_nil(spans.first.get_tag('active_record.db.cached')) # Assert cached flag set correctly on second query - assert_equal('true', spans.last.get_tag('rails.db.cached')) + assert_equal('true', spans.last.get_tag('active_record.db.cached')) end end @@ -86,8 +119,8 @@ class DatabaseTracingTest < ActiveSupport::TestCase # make the query and assert the proper spans Article.count - spans = @tracer.writer.spans() - assert_equal(spans.length, 1) + spans = @tracer.writer.spans + assert_equal(1, spans.length) span = spans.first assert_equal(span.service, 'customer-db') diff --git a/test/contrib/rails/rack_middleware_test.rb b/test/contrib/rails/rack_middleware_test.rb index 70a574b368d..9433cd7eee2 100644 --- a/test/contrib/rails/rack_middleware_test.rb +++ b/test/contrib/rails/rack_middleware_test.rb @@ -37,7 +37,7 @@ class FullStackTest < ActionDispatch::IntegrationTest # spans are sorted alphabetically, and ... controller names start # either by m or p (MySQL or PostGreSQL) so the database span is always # the first one. Would fail with an adapter named z-something. - if Datadog::Contrib::Rails::Patcher.active_record_instantiation_tracing_supported? + if Datadog::Contrib::ActiveRecord::Patcher.instantiation_tracing_supported? assert_equal(spans.length, 6) instantiation_span, database_span, request_span, controller_span, cache_span, render_span = spans else @@ -67,13 +67,13 @@ class FullStackTest < ActionDispatch::IntegrationTest assert_equal(database_span.name, "#{adapter_name}.query") assert_equal(database_span.span_type, 'sql') assert_equal(database_span.service, adapter_name) - assert_equal(database_span.get_tag('rails.db.vendor'), adapter_name) - assert_nil(database_span.get_tag('rails.db.cached')) + assert_equal(adapter_name, database_span.get_tag('active_record.db.vendor')) + assert_nil(database_span.get_tag('active_record.db.cached')) assert_includes(database_span.resource, 'SELECT') assert_includes(database_span.resource, 'FROM') assert_includes(database_span.resource, 'articles') - if Datadog::Contrib::Rails::Patcher.active_record_instantiation_tracing_supported? + if Datadog::Contrib::ActiveRecord::Patcher.instantiation_tracing_supported? assert_equal(instantiation_span.name, 'active_record.instantiation') assert_equal(instantiation_span.span_type, 'custom') assert_equal(instantiation_span.service, Datadog.configuration[:rails][:service_name]) diff --git a/test/contrib/rails/tracer_test.rb b/test/contrib/rails/tracer_test.rb index 3612ed03c3c..4d80b944034 100644 --- a/test/contrib/rails/tracer_test.rb +++ b/test/contrib/rails/tracer_test.rb @@ -28,7 +28,7 @@ class TracerTest < ActionDispatch::IntegrationTest test 'a default service and database should be properly set' do services = Datadog.configuration[:rails][:tracer].services Datadog::Contrib::Rails::Framework.setup - adapter_name = get_adapter_name() + adapter_name = get_adapter_name refute_equal(adapter_name, 'defaultdb') assert_equal( { @@ -36,7 +36,7 @@ class TracerTest < ActionDispatch::IntegrationTest 'app' => 'rails', 'app_type' => 'web' }, "#{app_name}-#{adapter_name}" => { - 'app' => adapter_name, 'app_type' => 'db' + 'app' => 'active_record', 'app_type' => 'db' }, "#{app_name}-cache" => { 'app' => 'rails', 'app_type' => 'cache' @@ -49,7 +49,6 @@ class TracerTest < ActionDispatch::IntegrationTest test 'database service can be changed by user' do update_config(:database_service, 'customer-db') tracer = Datadog.configuration[:rails][:tracer] - adapter_name = get_adapter_name() assert_equal( { @@ -57,7 +56,7 @@ class TracerTest < ActionDispatch::IntegrationTest 'app' => 'rails', 'app_type' => 'web' }, 'customer-db' => { - 'app' => adapter_name, 'app_type' => 'db' + 'app' => 'active_record', 'app_type' => 'db' }, "#{app_name}-cache" => { 'app' => 'rails', 'app_type' => 'cache' @@ -81,7 +80,7 @@ class TracerTest < ActionDispatch::IntegrationTest 'app' => 'rails', 'app_type' => 'web' }, "#{app_name}-#{adapter_name}" => { - 'app' => adapter_name, 'app_type' => 'db' + 'app' => 'active_record', 'app_type' => 'db' }, "#{app_name}-cache" => { 'app' => 'rails', 'app_type' => 'cache' @@ -102,7 +101,7 @@ class TracerTest < ActionDispatch::IntegrationTest 'app' => 'rails', 'app_type' => 'web' }, "#{app_name}-#{adapter_name}" => { - 'app' => adapter_name, 'app_type' => 'db' + 'app' => 'active_record', 'app_type' => 'db' }, 'service-cache' => { 'app' => 'rails', 'app_type' => 'cache' From 412b45cebd17a89d26ae2036a6d98f438ea8c8cd Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 9 Apr 2018 15:06:45 -0400 Subject: [PATCH 62/72] Refactored: ActiveRecord to use ActiveSupport::Notifications subscriptions. --- lib/ddtrace/contrib/active_record/patcher.rb | 84 ++++++++++--------- .../notifications/subscription.rb | 2 +- lib/ddtrace/contrib/racecar/patcher.rb | 9 +- 3 files changed, 54 insertions(+), 41 deletions(-) diff --git a/lib/ddtrace/contrib/active_record/patcher.rb b/lib/ddtrace/contrib/active_record/patcher.rb index 69ce24d391c..d05f04c87e7 100644 --- a/lib/ddtrace/contrib/active_record/patcher.rb +++ b/lib/ddtrace/contrib/active_record/patcher.rb @@ -1,6 +1,7 @@ require 'ddtrace/ext/sql' require 'ddtrace/ext/app_types' require 'ddtrace/contrib/active_record/utils' +require 'ddtrace/contrib/active_support/notifications/subscriber' module Datadog module Contrib @@ -8,6 +9,11 @@ module ActiveRecord # Patcher enables patching of 'active_record' module. module Patcher include Base + include ActiveSupport::Notifications::Subscriber + + NAME_SQL = 'sql.active_record'.freeze + NAME_INSTANTIATION = 'instantiation.active_record'.freeze + register_as :active_record, auto_patch: false option :service_name, depends_on: [:tracer] do |value| (value || Utils.adapter_name).tap do |v| @@ -15,10 +21,39 @@ module Patcher end end option :orm_service_name - option :tracer, default: Datadog.tracer + option :tracer, default: Datadog.tracer do |value| + (value || Datadog.tracer).tap do |v| + # Make sure to update tracers of all subscriptions + subscriptions.each do |subscription| + subscription.tracer = v + end + end + end @patched = false + on_subscribe do + # sql.active_record + subscribe( + self::NAME_SQL, # Event name + 'active_record.sql', # Span name + { service: get_option(:service_name) }, # Span options + get_option(:tracer), # Tracer + &method(:sql) # Handler + ) + + # instantiation.active_record + if instantiation_tracing_supported? + subscribe( + self::NAME_INSTANTIATION, # Event name + 'active_record.instantiation', # Span name + { service: get_option(:service_name) }, # Span options + get_option(:tracer), # Tracer + &method(:instantiation) # Handler + ) + end + end + module_function # patched? tells whether patch has been successfully applied @@ -29,7 +64,7 @@ def patched? def patch if !@patched && defined?(::ActiveRecord) begin - patch_active_record + subscribe! @patched = true rescue StandardError => e Datadog::Tracer.log.error("Unable to apply Active Record integration: #{e}") @@ -39,62 +74,33 @@ def patch @patched end - def patch_active_record - # subscribe when the active record query has been processed - ::ActiveSupport::Notifications.subscribe('sql.active_record') do |*args| - sql(*args) - end - - if instantiation_tracing_supported? - # subscribe when the active record instantiates objects - ::ActiveSupport::Notifications.subscribe('instantiation.active_record') do |*args| - instantiation(*args) - end - end - end - def instantiation_tracing_supported? Gem.loaded_specs['activerecord'] \ && Gem.loaded_specs['activerecord'].version >= Gem::Version.new('4.2') end - def self.sql(_name, start, finish, _id, payload) + def sql(span, event, _id, payload) connection_config = Utils.connection_config(payload[:connection_id]) - - span = get_option(:tracer).trace( - "#{connection_config[:adapter_name]}.query", - resource: payload.fetch(:sql), - service: get_option(:service_name), - span_type: Datadog::Ext::SQL::TYPE - ) + span.name = "#{connection_config[:adapter_name]}.query" + span.service = get_option(:service_name) + span.resource = payload.fetch(:sql) + span.span_type = Datadog::Ext::SQL::TYPE # Find out if the SQL query has been cached in this request. This meta is really # helpful to users because some spans may have 0ns of duration because the query # is simply cached from memory, so the notification is fired with start == finish. cached = payload[:cached] || (payload[:name] == 'CACHE') - # the span should have the query ONLY in the Resource attribute, - # so that the ``sql.query`` tag will be set in the agent with an - # obfuscated version - span.span_type = Datadog::Ext::SQL::TYPE span.set_tag('active_record.db.vendor', connection_config[:adapter_name]) span.set_tag('active_record.db.name', connection_config[:database_name]) span.set_tag('active_record.db.cached', cached) if cached span.set_tag('out.host', connection_config[:adapter_host]) span.set_tag('out.port', connection_config[:adapter_port]) - span.start_time = start - span.finish(finish) rescue StandardError => e Datadog::Tracer.log.debug(e.message) end - def self.instantiation(_name, start, finish, _id, payload) - span = get_option(:tracer).trace( - 'active_record.instantiation', - resource: payload.fetch(:class_name), - span_type: 'custom' - ) - + def instantiation(span, event, _id, payload) # Inherit service name from parent, if available. span.service = if get_option(:orm_service_name) get_option(:orm_service_name) @@ -104,10 +110,10 @@ def self.instantiation(_name, start, finish, _id, payload) 'active_record' end + span.resource = payload.fetch(:class_name) + span.span_type = 'custom' span.set_tag('active_record.instantiation.class_name', payload.fetch(:class_name)) span.set_tag('active_record.instantiation.record_count', payload.fetch(:record_count)) - span.start_time = start - span.finish(finish) rescue StandardError => e Datadog::Tracer.log.debug(e.message) end diff --git a/lib/ddtrace/contrib/active_support/notifications/subscription.rb b/lib/ddtrace/contrib/active_support/notifications/subscription.rb index f500cf413ae..1bc34f9b02d 100644 --- a/lib/ddtrace/contrib/active_support/notifications/subscription.rb +++ b/lib/ddtrace/contrib/active_support/notifications/subscription.rb @@ -4,7 +4,7 @@ module ActiveSupport module Notifications # An ActiveSupport::Notification subscription that wraps events with tracing. class Subscription - attr_reader \ + attr_accessor \ :tracer, :span_name, :options diff --git a/lib/ddtrace/contrib/racecar/patcher.rb b/lib/ddtrace/contrib/racecar/patcher.rb index f7ff4f7fd9a..71df0968bdf 100644 --- a/lib/ddtrace/contrib/racecar/patcher.rb +++ b/lib/ddtrace/contrib/racecar/patcher.rb @@ -12,8 +12,15 @@ module Patcher NAME_MESSAGE = 'racecar.message'.freeze NAME_BATCH = 'racecar.batch'.freeze register_as :racecar - option :tracer, default: Datadog.tracer option :service_name, default: 'racecar' + option :tracer, default: Datadog.tracer do |value| + (value || Datadog.tracer).tap do |v| + # Make sure to update tracers of all subscriptions + subscriptions.each do |subscription| + subscription.tracer = v + end + end + end on_subscribe do # Subscribe to single messages From 0478c38804ab23f4262d22ba4861ab936d4671af Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 9 Apr 2018 15:52:55 -0400 Subject: [PATCH 63/72] Fixed: Rails patcher missing constant. --- lib/ddtrace/contrib/rails/patcher.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ddtrace/contrib/rails/patcher.rb b/lib/ddtrace/contrib/rails/patcher.rb index 7f2ce690780..384de1d44a0 100644 --- a/lib/ddtrace/contrib/rails/patcher.rb +++ b/lib/ddtrace/contrib/rails/patcher.rb @@ -1,3 +1,5 @@ +require 'ddtrace/contrib/rails/utils' + module Datadog module Contrib module Rails From fcdbf2001f7d3be64a60364a099171258dab27b6 Mon Sep 17 00:00:00 2001 From: David Elner Date: Mon, 9 Apr 2018 16:22:41 -0400 Subject: [PATCH 64/72] Fixed: Broken test for instantiation spans. --- test/contrib/rails/database_test.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/contrib/rails/database_test.rb b/test/contrib/rails/database_test.rb index 07934e07059..9daf2dec3a7 100644 --- a/test/contrib/rails/database_test.rb +++ b/test/contrib/rails/database_test.rb @@ -80,7 +80,10 @@ class DatabaseTracingTest < ActiveSupport::TestCase end spans = @tracer.writer.spans assert_equal(3, spans.length) - instantiation_span, parent_span, _query_span = spans + parent_span = spans.find { |s| s.name == 'parent.span' } + instantiation_span = spans.find { |s| s.name == 'active_record.instantiation' } + + assert_equal(parent_span.service, 'parent-service') assert_equal(instantiation_span.name, 'active_record.instantiation') assert_equal(instantiation_span.span_type, 'custom') From e5d692c3256f685302e3b855d5ff060df02ab34b Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 10 Apr 2018 10:48:48 -0400 Subject: [PATCH 65/72] Fixed: Service info with wrong DB adapter when Rails integration is activated prematurely. --- lib/ddtrace/contrib/rails/framework.rb | 32 ++++++++++++++++-------- lib/ddtrace/contrib/rails/patcher.rb | 8 +++--- test/contrib/rails/apps/models.rb | 4 +-- test/contrib/rails/rails_sidekiq_test.rb | 11 ++++---- 4 files changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/ddtrace/contrib/rails/framework.rb b/lib/ddtrace/contrib/rails/framework.rb index 95568efdcb9..43cfd13693e 100644 --- a/lib/ddtrace/contrib/rails/framework.rb +++ b/lib/ddtrace/contrib/rails/framework.rb @@ -21,22 +21,26 @@ module Rails module Framework # configure Datadog settings def self.setup - config = Datadog.configuration[:rails] - config[:service_name] ||= Utils.app_name - tracer = config[:tracer] + config = config_with_defaults activate_rack!(config) activate_active_record!(config) - - config[:controller_service] ||= config[:service_name] - config[:cache_service] ||= "#{config[:service_name]}-cache" - - tracer.set_service_info(config[:controller_service], 'rails', Ext::AppTypes::WEB) - tracer.set_service_info(config[:cache_service], 'rails', Ext::AppTypes::CACHE) + set_service_info!(config) # By default, default service would be guessed from the script # being executed, but here we know better, get it from Rails config. - tracer.default_service = config[:service_name] + config[:tracer].default_service = config[:service_name] + end + + def self.config_with_defaults + # We set defaults here instead of in the patcher because we need to wait + # for the Rails application to be fully initialized. + Datadog.configuration[:rails].tap do |config| + config[:service_name] ||= Utils.app_name + config[:database_service] ||= "#{config[:service_name]}-#{Contrib::ActiveRecord::Utils.adapter_name}" + config[:controller_service] ||= config[:service_name] + config[:cache_service] ||= "#{config[:service_name]}-cache" + end end def self.activate_rack!(config) @@ -51,12 +55,20 @@ def self.activate_rack!(config) end def self.activate_active_record!(config) + return unless defined?(::ActiveRecord) + Datadog.configuration.use( :active_record, service_name: config[:database_service], tracer: config[:tracer] ) end + + def self.set_service_info!(config) + tracer = config[:tracer] + tracer.set_service_info(config[:controller_service], 'rails', Ext::AppTypes::WEB) + tracer.set_service_info(config[:cache_service], 'rails', Ext::AppTypes::CACHE) + end end end end diff --git a/lib/ddtrace/contrib/rails/patcher.rb b/lib/ddtrace/contrib/rails/patcher.rb index 384de1d44a0..19e4f64be10 100644 --- a/lib/ddtrace/contrib/rails/patcher.rb +++ b/lib/ddtrace/contrib/rails/patcher.rb @@ -8,15 +8,13 @@ module Patcher include Base register_as :rails, auto_patch: true - option :service_name do |value| - value || Utils.app_name - end + option :service_name option :controller_service option :cache_service option :database_service, depends_on: [:service_name] do |value| - (value || "#{get_option(:service_name)}-#{Contrib::ActiveRecord::Utils.adapter_name}").tap do |v| + value.tap do # Update ActiveRecord service name too - Datadog.configuration[:active_record][:service_name] = v + Datadog.configuration[:active_record][:service_name] = value end end option :middleware_names, default: false diff --git a/test/contrib/rails/apps/models.rb b/test/contrib/rails/apps/models.rb index 16443932b29..5ccc9217cd5 100644 --- a/test/contrib/rails/apps/models.rb +++ b/test/contrib/rails/apps/models.rb @@ -12,7 +12,7 @@ class Article < ApplicationRecord # MySQL JDBC drivers require that, otherwise we get a # "Table '?' already exists" error begin - Article.count() + Article.count rescue ActiveRecord::StatementInvalid logger.info 'Executing database migrations' ActiveRecord::Schema.define(version: 20161003090450) do @@ -27,4 +27,4 @@ class Article < ApplicationRecord end # force an access to prevent extra spans during tests -Article.count() +Article.count diff --git a/test/contrib/rails/rails_sidekiq_test.rb b/test/contrib/rails/rails_sidekiq_test.rb index dd2c04e0a7f..e9708468fcc 100644 --- a/test/contrib/rails/rails_sidekiq_test.rb +++ b/test/contrib/rails/rails_sidekiq_test.rb @@ -10,7 +10,7 @@ class RailsSidekiqTest < ActionController::TestCase setup do # don't pollute the global tracer @original_tracer = Datadog.configuration[:rails][:tracer] - @tracer = get_test_tracer() + @tracer = get_test_tracer Datadog.configuration[:rails][:tracer] = @tracer # configure Sidekiq @@ -33,13 +33,13 @@ class RailsSidekiqTest < ActionController::TestCase class EmptyWorker include Sidekiq::Worker - def perform(); end + def perform; end end test 'Sidekiq middleware uses Rails configuration if available' do @tracer.configure(enabled: false, debug: true, host: 'tracer.example.com', port: 7777) Datadog::Contrib::Rails::Framework.setup - db_adapter = get_adapter_name() + db_adapter = get_adapter_name # add Sidekiq middleware Sidekiq::Testing.server_middleware do |chain| @@ -47,15 +47,14 @@ def perform(); end end # do something to force middleware execution - EmptyWorker.perform_async() - + EmptyWorker.perform_async assert_equal( @tracer.services, app_name => { 'app' => 'rails', 'app_type' => 'web' }, "#{app_name}-#{db_adapter}" => { - 'app' => db_adapter, 'app_type' => 'db' + 'app' => 'active_record', 'app_type' => 'db' }, "#{app_name}-cache" => { 'app' => 'rails', 'app_type' => 'cache' From c415919c18aee24577077322ca7aac2d9226db73 Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 10 Apr 2018 16:07:53 -0400 Subject: [PATCH 66/72] Changed: Don't normalize 'mysql2' to 'mysql' (will do this later) --- lib/ddtrace/contrib/active_record/utils.rb | 2 -- spec/ddtrace/contrib/active_record/tracer_spec.rb | 10 +++++----- spec/ddtrace/contrib/active_record/utils_spec.rb | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/ddtrace/contrib/active_record/utils.rb b/lib/ddtrace/contrib/active_record/utils.rb index f65731d18a8..00315e463d5 100644 --- a/lib/ddtrace/contrib/active_record/utils.rb +++ b/lib/ddtrace/contrib/active_record/utils.rb @@ -8,8 +8,6 @@ def self.normalize_vendor(vendor) case vendor when nil 'defaultdb' - when 'mysql2' - 'mysql' when 'postgresql' 'postgres' when 'sqlite3' diff --git a/spec/ddtrace/contrib/active_record/tracer_spec.rb b/spec/ddtrace/contrib/active_record/tracer_spec.rb index ff186063e66..cfef4d4ae39 100644 --- a/spec/ddtrace/contrib/active_record/tracer_spec.rb +++ b/spec/ddtrace/contrib/active_record/tracer_spec.rb @@ -30,14 +30,14 @@ # expect service and trace is sent expect(spans.size).to eq(1) - expect(services['mysql']).to eq({'app'=>'active_record', 'app_type'=>'db'}) + expect(services['mysql2']).to eq({'app'=>'active_record', 'app_type'=>'db'}) span = spans[0] - expect(span.service).to eq('mysql') - expect(span.name).to eq('mysql.query') + expect(span.service).to eq('mysql2') + expect(span.name).to eq('mysql2.query') expect(span.span_type).to eq('sql') expect(span.resource.strip).to eq('SELECT COUNT(*) FROM `articles`') - expect(span.get_tag('active_record.db.vendor')).to eq('mysql') + expect(span.get_tag('active_record.db.vendor')).to eq('mysql2') expect(span.get_tag('active_record.db.name')).to eq('mysql') expect(span.get_tag('active_record.db.cached')).to eq(nil) expect(span.get_tag('out.host')).to eq('127.0.0.1') @@ -55,7 +55,7 @@ context 'is not set' do let(:configuration_options) { super().merge({ service_name: nil }) } - it { expect(query_span.service).to eq('mysql') } + it { expect(query_span.service).to eq('mysql2') } end context 'is set' do diff --git a/spec/ddtrace/contrib/active_record/utils_spec.rb b/spec/ddtrace/contrib/active_record/utils_spec.rb index dd656b83510..eccef572ff6 100644 --- a/spec/ddtrace/contrib/active_record/utils_spec.rb +++ b/spec/ddtrace/contrib/active_record/utils_spec.rb @@ -20,7 +20,7 @@ context 'mysql2' do let(:value) { 'mysql2' } - it { is_expected.to eq('mysql') } + it { is_expected.to eq('mysql2') } end context 'postgresql' do From 469acc8e78703b123bcf61147f2890f37f3d8e58 Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 10 Apr 2018 16:04:13 -0400 Subject: [PATCH 67/72] Added: 0.12.0.rc1 to CHANGELOG.md --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9d6160d89f..acd7f4b314c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,38 @@ ## [Unreleased (beta)] +## [0.12.0.rc1] - 2018-04-11 + +Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.12.0.rc1 + +Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.11.4...v0.12.0.rc1 + +### Added +- GraphQL integration (supporting graphql 1.7.9+) (#295) +- ActiveRecord object instantiation tracing (#311, #334) +- Subscriber module for ActiveSupport::Notifications tracing (#324, #380, #390, #395) (@dasch) +- HTTP quantization module (#384) +- Partial flushing option to tracer (#247, #397) + +### Changed +- Rack applies URL quantization by default (#371) +- Elasticsearch applies body quantization by default (#362) +- Context for a single trace now has hard limit of 100,000 spans (#247) +- Tags with `rails.db.x` to `active_record.db.x` instead (#396) + +### Fixed +- Loading the ddtrace library after Rails has fully initialized can result in load errors. (#357) +- Some scenarios where `middleware_names` could result in bad resource names (#354) +- ActionController instrumentation conflicting with some gems that monkey patch Rails (#391) + +### Deprecated +- Use of `:datadog_rack_request_span` variable in favor of `'datadog.rack_request_span'` in Rack. (#365, #392) + +### Refactored +- Racecar to use ActiveSupport::Notifications Subscriber module (#381) +- Rails to use ActiveRecord integration instead of its own implementation (#396) +- ActiveRecord to use ActiveSupport::Notifications Subscriber module (#396) + ## [0.12.0.beta2] - 2018-02-28 Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.12.0.beta2 @@ -221,8 +253,9 @@ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.3.1 Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1 -[Unreleased (stable)]: https://github.com/DataDog/dd-trace-rb/compare/v0.11.3...master -[Unreleased (beta)]: https://github.com/DataDog/dd-trace-rb/compare/v0.12.0.beta2...0.12-dev +[Unreleased (stable)]: https://github.com/DataDog/dd-trace-rb/compare/v0.11.4...master +[Unreleased (beta)]: https://github.com/DataDog/dd-trace-rb/compare/v0.12.0.rc1...0.12-dev +[0.12.0.rc1]: https://github.com/DataDog/dd-trace-rb/compare/v0.11.4...v0.12.0.rc1 [0.12.0.beta2]: https://github.com/DataDog/dd-trace-rb/compare/v0.12.0.beta1...v0.12.0.beta2 [0.12.0.beta1]: https://github.com/DataDog/dd-trace-rb/compare/v0.11.2...v0.12.0.beta1 [0.11.4]: https://github.com/DataDog/dd-trace-rb/compare/v0.11.3...v0.11.4 From 489b2fb323694ed8b51a861f14c7c7b8f3f9e800 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 11 Apr 2018 10:12:19 -0400 Subject: [PATCH 68/72] bumping version 0.12.0.beta2 => 0.12.0.rc1 --- lib/ddtrace/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ddtrace/version.rb b/lib/ddtrace/version.rb index f4cd0fc4686..d3e6f960273 100644 --- a/lib/ddtrace/version.rb +++ b/lib/ddtrace/version.rb @@ -3,7 +3,7 @@ module VERSION MAJOR = 0 MINOR = 12 PATCH = 0 - PRE = 'beta2'.freeze + PRE = 'rc1'.freeze STRING = [MAJOR, MINOR, PATCH, PRE].compact.join('.') end From 59379aa606a676a88bc2a599b5805c024f9d15e9 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 11 Apr 2018 16:34:15 -0400 Subject: [PATCH 69/72] Removed: Deprecation comment for Rack request span. --- lib/ddtrace/contrib/rack/middlewares.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/ddtrace/contrib/rack/middlewares.rb b/lib/ddtrace/contrib/rack/middlewares.rb index 40e96a211d9..d071072c9e9 100644 --- a/lib/ddtrace/contrib/rack/middlewares.rb +++ b/lib/ddtrace/contrib/rack/middlewares.rb @@ -40,7 +40,6 @@ def call(env) env[RACK_REQUEST_SPAN] = request_span # TODO: For backwards compatibility; this attribute is deprecated. - # Will be removed in version 0.13.0. env[:datadog_rack_request_span] = env[RACK_REQUEST_SPAN] # Add deprecation warnings From 754bedc558ae7538a513a2b239a899b2245527e0 Mon Sep 17 00:00:00 2001 From: David Elner Date: Thu, 12 Apr 2018 15:35:03 -0400 Subject: [PATCH 70/72] Changed: Distributed tracing documentation to explain more details. --- docs/GettingStarted.md | 165 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 151 insertions(+), 14 deletions(-) diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index ebec956d338..a94ea21507d 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -885,21 +885,139 @@ span.context.sampling_priority = Datadog::Ext::Priority::USER_KEEP ### Distributed Tracing -To trace requests across hosts, the spans on the secondary hosts must be linked together by setting ``trace_id`` and ``parent_id``: +Distributed tracing allows traces to be propagated across multiple instrumented applications, so that a request can be presented as a single trace, rather than a separate trace per service. + +To trace requests across application boundaries, the following must be propagated between each application: + +| Property | Type | Description | +| --------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------- | +| **Trace ID** | Integer | ID of the trace. This value should be the same across all requests that belong to the same trace. | +| **Parent Span ID** | Integer | ID of the span in the service originating the request. This value will always be different for each request within a trace. | +| **Sampling Priority** | Integer | Sampling priority level for the trace. This value should be the same across all requests that belong to the same trace. | + +Such propagation can be visualized as: + +``` +Service A: + Trace ID: 100000000000000001 + Parent ID: 0 + Span ID: 100000000000000123 + Priority: 1 + + | + | Service B Request: + | Metadata: + | Trace ID: 100000000000000001 + | Parent ID: 100000000000000123 + | Priority: 1 + | + V + +Service B: + Trace ID: 100000000000000001 + Parent ID: 100000000000000123 + Span ID: 100000000000000456 + Priority: 1 + + | + | Service C Request: + | Metadata: + | Trace ID: 100000000000000001 + | Parent ID: 100000000000000456 + | Priority: 1 + | + V + +Service C: + Trace ID: 100000000000000001 + Parent ID: 100000000000000456 + Span ID: 100000000000000789 + Priority: 1 +``` + +**Via HTTP** + +For HTTP requests between instrumented applications, this trace metadata is propagated by use of HTTP Request headers: + +| Property | Type | HTTP Header name | +| --------------------- | ------- | ----------------------------- | +| **Trace ID** | Integer | `x-datadog-trace-id` | +| **Parent Span ID** | Integer | `x-datadog-parent-id` | +| **Sampling Priority** | Integer | `x-datadog-sampling-priority` | + +Such that: + +``` +Service A: + Trace ID: 100000000000000001 + Parent ID: 0 + Span ID: 100000000000000123 + Priority: 1 + + | + | Service B HTTP Request: + | Headers: + | x-datadog-trace-id: 100000000000000001 + | x-datadog-parent-id: 100000000000000123 + | x-datadog-sampling-priority: 1 + | + V + +Service B: + Trace ID: 100000000000000001 + Parent ID: 100000000000000123 + Span ID: 100000000000000456 + Priority: 1 + + | + | Service B HTTP Request: + | Headers: + | x-datadog-trace-id: 100000000000000001 + | x-datadog-parent-id: 100000000000000456 + | x-datadog-sampling-priority: 1 + | + V + +Service C: + Trace ID: 100000000000000001 + Parent ID: 100000000000000456 + Span ID: 100000000000000789 + Priority: 1 +``` + +**Activating distributed tracing for integrations** + +Many integrations included in `ddtrace` support distributed tracing. Distributed tracing is disabled by default, but can be activated via configuration settings. + +- If your application receives requests from services with distributed tracing activated, you must activate distributed tracing on the integrations that handle these requests (e.g. Rails) +- If your application send requests to services with distributed tracing activated, you must activate distributed tracing on the integrations that send these requests (e.g. Faraday) +- If your application both sends and receives requests implementing distributed tracing, it must activate all integrations which handle these requests. + +For more details on how to activate distributed tracing for integrations, see their documentation: + +- [Faraday](#faraday) +- [Net/HTTP](#nethttp) +- [Rack](#rack) +- [Rails](#rails) +- [Sinatra](#sinatra) + +**Implementing distributed tracing manually** + +If your application either receives or sends requests through non-instrumented code, you must manually configure distributed tracing. + +Spans on receiving applications must be linked to their parent trace by setting `trace_id` and `parent_id`: ```ruby def request_on_secondary_host(parent_trace_id, parent_span_id) - tracer.trace('web.request') do |span| - span.parent_id = parent_span_id - span.trace_id = parent_trace_id + tracer.trace('web.request') do |span| + span.parent_id = parent_span_id + span.trace_id = parent_trace_id - # perform user code - end + # perform user code + end end ``` -Users can pass along the ``parent_trace_id`` and ``parent_span_id`` via whatever method best matches the RPC framework. - Below is an example using Net/HTTP and Sinatra, where we bypass the integrations to demo how distributed tracing works. On the client: @@ -934,19 +1052,38 @@ get '/' do parent_span_id = request.env['HTTP_X_DATADOG_PARENT_ID'] Datadog.tracer.trace('web.work') do |span| - if parent_trace_id && parent_span_id - span.trace_id = parent_trace_id.to_i - span.parent_id = parent_span_id.to_i - end + if parent_trace_id && parent_span_id + span.trace_id = parent_trace_id.to_i + span.parent_id = parent_span_id.to_i + end 'Hello world!' end end ``` -[Rack](#rack) and [Net/HTTP](#nethttp) have experimental support for this, they can send and receive these headers automatically and tie spans together automatically, provided you pass a ``:distributed_tracing`` option set to ``true``. +**Using the HTTP propagator** + +To make the process of propagating this metadata easier, you can use the `Datadog::HTTPPropagator` module. -Distributed tracing is disabled by default. +On the client: + +```ruby +Datadog.tracer.trace('web.call') do |span| + # Inject span context into headers (`env` must be a Hash) + Datadog::HTTPPropagator.inject!(span.context, env) +end +``` + +On the server: + +```ruby +Datadog.tracer.trace('web.work') do |span| + # Build a context from headers (`env` must be a Hash) + context = HTTPPropagator.extract(request.env) + Datadog.tracer.provider.context = context if context.trace_id +end +``` ### Processing Pipeline From b4eb952015d0c712cb3f1297ededc5afb258557c Mon Sep 17 00:00:00 2001 From: David Elner Date: Fri, 13 Apr 2018 13:24:40 -0400 Subject: [PATCH 71/72] Removed: Manual distributed tracing description. --- docs/GettingStarted.md | 61 ------------------------------------------ 1 file changed, 61 deletions(-) diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md index a94ea21507d..b5f9d7cbe2d 100644 --- a/docs/GettingStarted.md +++ b/docs/GettingStarted.md @@ -1001,67 +1001,6 @@ For more details on how to activate distributed tracing for integrations, see th - [Rails](#rails) - [Sinatra](#sinatra) -**Implementing distributed tracing manually** - -If your application either receives or sends requests through non-instrumented code, you must manually configure distributed tracing. - -Spans on receiving applications must be linked to their parent trace by setting `trace_id` and `parent_id`: - -```ruby -def request_on_secondary_host(parent_trace_id, parent_span_id) - tracer.trace('web.request') do |span| - span.parent_id = parent_span_id - span.trace_id = parent_trace_id - - # perform user code - end -end -``` - -Below is an example using Net/HTTP and Sinatra, where we bypass the integrations to demo how distributed tracing works. - -On the client: - -```ruby -require 'net/http' -require 'ddtrace' - -uri = URI('http://localhost:4567/') - -Datadog.tracer.trace('web.call') do |span| - req = Net::HTTP::Get.new(uri) - req['x-datadog-trace-id'] = span.trace_id.to_s - req['x-datadog-parent-id'] = span.span_id.to_s - - response = Net::HTTP.start(uri.hostname, uri.port) do |http| - http.request(req) - end - - puts response.body -end -``` - -On the server: - -```ruby -require 'sinatra' -require 'ddtrace' - -get '/' do - parent_trace_id = request.env['HTTP_X_DATADOG_TRACE_ID'] - parent_span_id = request.env['HTTP_X_DATADOG_PARENT_ID'] - - Datadog.tracer.trace('web.work') do |span| - if parent_trace_id && parent_span_id - span.trace_id = parent_trace_id.to_i - span.parent_id = parent_span_id.to_i - end - - 'Hello world!' - end -end -``` - **Using the HTTP propagator** To make the process of propagating this metadata easier, you can use the `Datadog::HTTPPropagator` module. From 6ba8d072b5cae2c83b1349f4f85c729c1b6bf1fe Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 8 May 2018 11:17:40 -0400 Subject: [PATCH 72/72] Bump to version 0.12.0 (#414) * Added: 0.12.0 to CHANGELOG * bumping version 0.12.0.rc1 => 0.12.0 --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++-- lib/ddtrace/version.rb | 2 +- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index acd7f4b314c..63bfb9e84f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,38 @@ ## [Unreleased (beta)] +## [0.12.0] - 2018-05-08 + +Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.12.0 + +Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.11.4...v0.12.0 + +### Added +- GraphQL integration (supporting graphql 1.7.9+) (#295) +- ActiveRecord object instantiation tracing (#311, #334) +- Subscriber module for ActiveSupport::Notifications tracing (#324, #380, #390, #395) (@dasch) +- HTTP quantization module (#384) +- Partial flushing option to tracer (#247, #397) + +### Changed +- Rack applies URL quantization by default (#371) +- Elasticsearch applies body quantization by default (#362) +- Context for a single trace now has hard limit of 100,000 spans (#247) +- Tags with `rails.db.x` to `active_record.db.x` instead (#396) + +### Fixed +- Loading the ddtrace library after Rails has fully initialized can result in load errors. (#357) +- Some scenarios where `middleware_names` could result in bad resource names (#354) +- ActionController instrumentation conflicting with some gems that monkey patch Rails (#391) + +### Deprecated +- Use of `:datadog_rack_request_span` variable in favor of `'datadog.rack_request_span'` in Rack. (#365, #392) + +### Refactored +- Racecar to use ActiveSupport::Notifications Subscriber module (#381) +- Rails to use ActiveRecord integration instead of its own implementation (#396) +- ActiveRecord to use ActiveSupport::Notifications Subscriber module (#396) + ## [0.12.0.rc1] - 2018-04-11 Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.12.0.rc1 @@ -253,8 +285,9 @@ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.3.1 Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1 -[Unreleased (stable)]: https://github.com/DataDog/dd-trace-rb/compare/v0.11.4...master -[Unreleased (beta)]: https://github.com/DataDog/dd-trace-rb/compare/v0.12.0.rc1...0.12-dev +[Unreleased (stable)]: https://github.com/DataDog/dd-trace-rb/compare/v0.12.0...master +[Unreleased (beta)]: https://github.com/DataDog/dd-trace-rb/compare/v0.12.0...0.13-dev +[0.12.0]: https://github.com/DataDog/dd-trace-rb/compare/v0.11.4...v0.12.0 [0.12.0.rc1]: https://github.com/DataDog/dd-trace-rb/compare/v0.11.4...v0.12.0.rc1 [0.12.0.beta2]: https://github.com/DataDog/dd-trace-rb/compare/v0.12.0.beta1...v0.12.0.beta2 [0.12.0.beta1]: https://github.com/DataDog/dd-trace-rb/compare/v0.11.2...v0.12.0.beta1 diff --git a/lib/ddtrace/version.rb b/lib/ddtrace/version.rb index d3e6f960273..343384c5412 100644 --- a/lib/ddtrace/version.rb +++ b/lib/ddtrace/version.rb @@ -3,7 +3,7 @@ module VERSION MAJOR = 0 MINOR = 12 PATCH = 0 - PRE = 'rc1'.freeze + PRE = nil STRING = [MAJOR, MINOR, PATCH, PRE].compact.join('.') end