Merge branch 'dev' into httprb-support

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2023-06-19 05:15:43 UTC using RuboCop version 1.52.1.
+# on 2023-07-14 05:55:13 UTC using RuboCop version 1.52.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -108,7 +108,7 @@ Layout/EmptyLinesAroundBlockBody:
     - 'test/newrelic_security/instrumentation-security/active_record/sqlite3_adapter/db/schema.rb'
     - 'test/newrelic_security/instrumentation-security/rails/rails_test.rb'
 
-# Offense count: 25
+# Offense count: 26
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines, beginning_only, ending_only
@@ -128,7 +128,7 @@ Layout/EmptyLinesAroundMethodBody:
 Layout/EmptyLinesAroundModuleBody:
   Enabled: false
 
-# Offense count: 30
+# Offense count: 33
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowForAlignment, AllowBeforeTrailingComments, ForceEqualSignAlignment.
 Layout/ExtraSpacing:
@@ -211,7 +211,7 @@ Layout/IndentationStyle:
 Layout/IndentationWidth:
   Enabled: false
 
-# Offense count: 86
+# Offense count: 98
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowDoxygenCommentStyle, AllowGemfileRubyComment.
 Layout/LeadingCommentSpace:
@@ -266,7 +266,7 @@ Layout/SpaceAroundKeyword:
   Exclude:
     - 'test/newrelic_security/instrumentation-security/curb/curb_test.rb'
 
-# Offense count: 551
+# Offense count: 552
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowForAlignment, EnforcedStyleForExponentOperator.
 # SupportedStylesForExponentOperator: space, no_space
@@ -354,14 +354,14 @@ Layout/SpaceInsideStringInterpolation:
   Exclude:
     - 'lib/newrelic_security/instrumentation-security/net_http/instrumentation.rb'
 
-# Offense count: 82
+# Offense count: 83
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: final_newline, final_blank_line
 Layout/TrailingEmptyLines:
   Enabled: false
 
-# Offense count: 492
+# Offense count: 504
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowInHeredoc.
 Layout/TrailingWhitespace:
@@ -410,7 +410,7 @@ Lint/EmptyWhen:
   Exclude:
     - 'lib/newrelic_security/agent/control/control_command.rb'
 
-# Offense count: 68
+# Offense count: 72
 # This cop supports safe autocorrection (--autocorrect).
 Lint/EnsureReturn:
   Enabled: false
@@ -465,7 +465,7 @@ Lint/RedundantStringCoercion:
   Exclude:
     - 'test/newrelic_security/instrumentation-security/nokogiri/nokogiri_test.rb'
 
-# Offense count: 19
+# Offense count: 20
 Lint/RescueException:
   Exclude:
     - 'lib/newrelic_security/agent/agent.rb'
@@ -493,7 +493,7 @@ Lint/SuppressedException:
     - 'test/newrelic_security/instrumentation-security/mysql2/mysql2_test.rb'
     - 'test/newrelic_security/instrumentation-security/pg/pg_test.rb'
 
-# Offense count: 5
+# Offense count: 6
 # This cop supports safe autocorrection (--autocorrect).
 Lint/ToJSON:
   Exclude:
@@ -502,6 +502,7 @@ Lint/ToJSON:
     - 'lib/newrelic_security/agent/control/exit_event.rb'
     - 'lib/newrelic_security/agent/control/fuzz_fail_event.rb'
     - 'lib/newrelic_security/agent/control/health_check.rb'
+    - 'lib/newrelic_security/agent/control/iast_data_transfer_request.rb'
 
 # Offense count: 4
 # This cop supports safe autocorrection (--autocorrect).
@@ -512,7 +513,7 @@ Lint/UnusedBlockArgument:
     - 'lib/newrelic_security/instrumentation-security/curb/instrumentation.rb'
     - 'test/newrelic_security/instrumentation-security/pty/pty_test.rb'
 
-# Offense count: 32
+# Offense count: 36
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods, IgnoreNotImplementedMethods.
 Lint/UnusedMethodArgument:
@@ -533,7 +534,7 @@ Lint/UnusedMethodArgument:
     - 'lib/newrelic_security/instrumentation-security/sqlite3/instrumentation.rb'
     - 'test/helpers/event_helper.rb'
 
-# Offense count: 19
+# Offense count: 20
 # This cop supports unsafe autocorrection (--autocorrect-all).
 Lint/UselessAssignment:
   Exclude:
@@ -546,12 +547,12 @@ Lint/UselessAssignment:
     - 'test/newrelic_security/instrumentation-security/active_record/sqlite3_adapter/sqlite3_adapter_test.rb'
     - 'test/newrelic_security/instrumentation-security/io/io_test.rb'
 
-# Offense count: 194
+# Offense count: 199
 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes.
 Metrics/AbcSize:
   Max: 218
 
-# Offense count: 6
+# Offense count: 8
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 # AllowedMethods: refine
 Metrics/BlockLength:
@@ -567,22 +568,22 @@ Metrics/BlockNesting:
 Metrics/ClassLength:
   Max: 490
 
-# Offense count: 25
+# Offense count: 29
 # Configuration parameters: AllowedMethods, AllowedPatterns.
 Metrics/CyclomaticComplexity:
   Max: 38
 
-# Offense count: 190
+# Offense count: 196
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 94
 
-# Offense count: 9
+# Offense count: 10
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ModuleLength:
   Max: 269
 
-# Offense count: 24
+# Offense count: 27
 # Configuration parameters: AllowedMethods, AllowedPatterns.
 Metrics/PerceivedComplexity:
   Max: 42
@@ -638,7 +639,7 @@ Naming/MemoizedInstanceVariableName:
   Exclude:
     - 'lib/newrelic_security/websocket-client-simple/websocket-ruby/lib/websocket/frame/handler/handler03.rb'
 
-# Offense count: 13
+# Offense count: 15
 # Configuration parameters: EnforcedStyle, AllowedPatterns.
 # SupportedStyles: snake_case, camelCase
 Naming/MethodName:
@@ -650,6 +651,7 @@ Naming/MethodName:
     - 'lib/newrelic_security/agent/control/fuzz_fail_event.rb'
     - 'lib/newrelic_security/agent/control/health_check.rb'
     - 'lib/newrelic_security/agent/control/iast_client.rb'
+    - 'lib/newrelic_security/agent/control/iast_data_transfer_request.rb'
     - 'lib/newrelic_security/agent/utils/agent_utils.rb'
     - 'test/newrelic_security/instrumentation-security/mongo/mongo_test.rb'
 
@@ -675,13 +677,13 @@ Naming/PredicateName:
     - 'lib/newrelic_security/agent/utils/agent_utils.rb'
     - 'lib/newrelic_security/newrelic-security-api/api.rb'
 
-# Offense count: 164
+# Offense count: 173
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: PreferredName.
 Naming/RescuedExceptionsVariableName:
   Enabled: false
 
-# Offense count: 123
+# Offense count: 127
 # Configuration parameters: EnforcedStyle, AllowedIdentifiers, AllowedPatterns.
 # SupportedStyles: snake_case, camelCase
 Naming/VariableName:
@@ -693,6 +695,7 @@ Naming/VariableName:
     - 'lib/newrelic_security/agent/control/fuzz_fail_event.rb'
     - 'lib/newrelic_security/agent/control/health_check.rb'
     - 'lib/newrelic_security/agent/control/iast_client.rb'
+    - 'lib/newrelic_security/agent/control/iast_data_transfer_request.rb'
     - 'lib/newrelic_security/agent/control/reflected_xss.rb'
     - 'test/helpers/event_helper.rb'
     - 'test/newrelic_security/instrumentation-security/rails/rails_test.rb'
@@ -731,7 +734,7 @@ Style/Alias:
   Exclude:
     - 'lib/newrelic_security/websocket-client-simple/event_emitter.rb'
 
-# Offense count: 11
+# Offense count: 17
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: always, conditionals
@@ -742,7 +745,7 @@ Style/AndOr:
     - 'lib/newrelic_security/instrumentation-security/active_record/sqlite3_adapter/instrumentation.rb'
     - 'lib/newrelic_security/websocket-client-simple/client.rb'
 
-# Offense count: 22
+# Offense count: 26
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle, ProceduralMethods, FunctionalMethods, AllowedMethods, AllowedPatterns, AllowBracesOnProceduralOneLiners, BracesRequiredMethods.
 # SupportedStyles: line_count_based, semantic, braces_for_chaining, always_braces
@@ -771,7 +774,7 @@ Style/CaseEquality:
   Exclude:
     - 'lib/newrelic_security/newrelic-security-api/api.rb'
 
-# Offense count: 175
+# Offense count: 176
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: nested, compact
@@ -825,7 +828,7 @@ Style/CommentedKeyword:
     - 'lib/newrelic_security/instrumentation-security/active_record/postgresql_adapter/prepend.rb'
     - 'test/newrelic_security/instrumentation-security/rails/rails_test.rb'
 
-# Offense count: 5
+# Offense count: 8
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle, SingleLineConditionsOnly, IncludeTernaryExpressions.
 # SupportedStyles: assign_to_condition, assign_inside_condition
@@ -845,7 +848,7 @@ Style/DefWithParentheses:
     - 'lib/newrelic_security/instrumentation-security/instrumentation_loader.rb'
     - 'test/helpers/agent_helper.rb'
 
-# Offense count: 155
+# Offense count: 156
 # Configuration parameters: AllowedConstants.
 Style/Documentation:
   Enabled: false
@@ -904,7 +907,7 @@ Style/For:
     - 'lib/newrelic_security/agent/control/reflected_xss.rb'
     - 'test/helpers/event_helper.rb'
 
-# Offense count: 116
+# Offense count: 117
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: always, always_true, never
@@ -1059,13 +1062,14 @@ Style/NegatedWhile:
     - 'lib/newrelic_security/websocket-client-simple/client.rb'
     - 'test/newrelic_security/instrumentation-security/httpclient/httpclient_test.rb'
 
-# Offense count: 2
+# Offense count: 3
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle, MinBodyLength.
 # SupportedStyles: skip_modifier_ifs, always
 Style/Next:
   Exclude:
     - 'lib/newrelic_security/agent/control/collector.rb'
+    - 'lib/newrelic_security/agent/control/iast_client.rb'
 
 # Offense count: 12
 # This cop supports safe autocorrection (--autocorrect).
@@ -1102,7 +1106,7 @@ Style/NumericLiterals:
     - 'test/newrelic_security/instrumentation-security/active_record/postgresql_adapter/db/schema.rb'
     - 'test/newrelic_security/instrumentation-security/active_record/sqlite3_adapter/db/schema.rb'
 
-# Offense count: 6
+# Offense count: 7
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: EnforcedStyle, AllowedMethods, AllowedPatterns.
 # SupportedStyles: predicate, comparison
@@ -1111,6 +1115,7 @@ Style/NumericPredicate:
     - 'spec/**/*'
     - 'lib/newrelic_security/agent/configuration/manager.rb'
     - 'lib/newrelic_security/agent/control/control_command.rb'
+    - 'lib/newrelic_security/agent/control/iast_client.rb'
     - 'lib/newrelic_security/agent/control/reflected_xss.rb'
     - 'lib/newrelic_security/instrumentation-security/kernel/instrumentation.rb'
     - 'lib/newrelic_security/instrumentation-security/sqlite3/instrumentation.rb'
@@ -1123,12 +1128,11 @@ Style/OptionalBooleanParameter:
     - 'lib/newrelic_security/agent/control/websocket_client.rb'
     - 'lib/newrelic_security/websocket-client-simple/client.rb'
 
-# Offense count: 2
+# Offense count: 1
 # This cop supports safe autocorrection (--autocorrect).
 Style/OrAssignment:
   Exclude:
     - 'lib/newrelic_security/agent.rb'
-    - 'lib/newrelic_security/agent/control/iast_client.rb'
 
 # Offense count: 2
 # This cop supports safe autocorrection (--autocorrect).
@@ -1256,7 +1260,7 @@ Style/RegexpLiteral:
   Exclude:
     - 'lib/newrelic_security/agent/control/reflected_xss.rb'
 
-# Offense count: 157
+# Offense count: 165
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: implicit, explicit
@@ -1326,7 +1330,7 @@ Style/StringConcatenation:
     - 'test/newrelic_security/instrumentation-security/pty/pty_test.rb'
     - 'test/newrelic_security/instrumentation-security/sqlite3/sqlite3_test.rb'
 
-# Offense count: 862
+# Offense count: 864
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
 # SupportedStyles: single_quotes, double_quotes
@@ -1402,7 +1406,7 @@ Style/ZeroLengthPredicate:
     - 'lib/newrelic_security/instrumentation-security/sqlite3/instrumentation.rb'
     - 'lib/newrelic_security/websocket-client-simple/websocket-ruby/lib/websocket/frame/handler/handler75.rb'
 
-# Offense count: 539
+# Offense count: 567
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns.
 # URISchemes: http, https

lib/newrelic_security/agent/agent.rb

@@ -2,14 +2,18 @@
 require 'newrelic_security/agent/control/websocket_client'
 require 'newrelic_security/agent/control/event_processor'
 require 'newrelic_security/agent/control/iast_client'
+require 'newrelic_security/agent/control/iast_data_transfer_request'
 require 'newrelic_security/agent/control/control_command'
+require 'newrelic_security/agent/control/fuzz_request'
 require 'newrelic_security/agent/control/reflected_xss'
 require 'newrelic_security/agent/control/http_context'
 require 'newrelic_security/agent/control/collector'
 require 'newrelic_security/agent/control/app_info'
 require 'newrelic_security/agent/control/health_check'
 require 'newrelic_security/agent/control/event'
+require 'newrelic_security/agent/control/critical_message'
 require 'newrelic_security/agent/control/event_counter'
+require 'newrelic_security/agent/control/event_stats'
 require 'newrelic_security/agent/control/exit_event'
 require 'newrelic_security/agent/control/fuzz_fail_event'
 require 'newrelic_security/instrumentation-security/instrumentation_loader'
@@ -19,7 +23,7 @@ module NewRelic::Security
   module Agent
     class Agent
 
-      attr_accessor :websocket_client, :event_processor, :iast_client, :http_request_count, :event_processed_count, :event_sent_count, :event_drop_count, :route_map, :status_logger
+      attr_accessor :websocket_client, :event_processor, :iast_client, :http_request_count, :event_processed_count, :event_sent_count, :event_drop_count, :route_map, :status_logger, :iast_event_stats, :rasp_event_stats, :exit_event_stats
 
       def initialize
         NewRelic::Security::Agent.config
@@ -33,6 +37,9 @@ def initialize
         @event_processed_count = NewRelic::Security::Agent::Control::EventCounter.new
         @event_sent_count = NewRelic::Security::Agent::Control::EventCounter.new
         @event_drop_count = NewRelic::Security::Agent::Control::EventCounter.new
+        @iast_event_stats = NewRelic::Security::Agent::Control::EventStats.new
+        @rasp_event_stats = NewRelic::Security::Agent::Control::EventStats.new
+        @exit_event_stats = NewRelic::Security::Agent::Control::EventStats.new
       end
 
       def init
@@ -68,6 +75,7 @@ def start_event_processor
 
       def start_iast_client
         @iast_client&.iast_dequeue_thread&.kill
+        @iast_client&.iast_data_transfer_request_processor_thread&.kill
         @iast_client = nil
         @iast_client = NewRelic::Security::Agent::Control::IASTClient.new
       end

lib/newrelic_security/agent/configuration/manager.rb

@@ -33,9 +33,10 @@ def initialize
           @cache[:'security.detection.deserialization.enabled'] = ::NewRelic::Agent.config[:'security.detection.deserialization.enabled']
           @cache[:framework] = detect_framework
           @cache[:'security.application_info.port'] = ::NewRelic::Agent.config[:'security.application_info.port'].to_i
+          @cache[:'security.request.body_limit'] = ::NewRelic::Agent.config[:'security.request.body_limit'].to_i > 0 ? ::NewRelic::Agent.config[:'security.request.body_limit'].to_i : 300
           @cache[:listen_port] = nil
           @cache[:app_root] = NewRelic::Security::Agent::Utils.app_root
-          @cache[:json_version] = :'1.0.1'
+          @cache[:json_version] = :'1.1.1'
 
           @environment_source = NewRelic::Security::Agent::Configuration::EnvironmentSource.new
           @server_source = NewRelic::Security::Agent::Configuration::ServerSource.new
@@ -98,6 +99,7 @@ def enable_security
           @cache[:enabled] = true
           NewRelic::Security::Agent.logger.info "Security Agent is now ACTIVE for #{NewRelic::Security::Agent.config[:uuid]}\n"
           NewRelic::Security::Agent.init_logger.info "Security Agent is now ACTIVE for #{NewRelic::Security::Agent.config[:uuid]}\n"
+          NewRelic::Security::Agent.agent.event_processor.send_critical_message("Security Agent is now ACTIVE for #{NewRelic::Security::Agent.config[:uuid]}", "INFO", caller_locations[0].to_s, Thread.current.name, nil)
         end
 
         private