Fix APM Disablement V1 (ASM Standalone)

.github/forced-tests-list.json

@@ -1,3 +1,5 @@
 {
-
-}
+  "SCA_STANDALONE_EXPERIMENTAL": [
+    "tests/appsec/test_asm_standalone.py::Test_SCAStandalone_Telemetry"
+  ]
+}

.github/workflows/system-tests.yml

@@ -263,7 +263,7 @@ jobs:
           - APPSEC_DISABLED
           - APPSEC_BLOCKING_FULL_DENYLIST
           - APPSEC_REQUEST_BLOCKING
-          - APPSEC_STANDALONE
+          - APPSEC_STANDALONE_EXPERIMENTAL
           - APPSEC_BLOCKING
         include:
           - library: ruby
@@ -320,6 +320,9 @@ jobs:
           - library: ruby
             app: rails70
             scenario: DEBUGGER_PII_REDACTION
+          - library: ruby
+            app: rails70
+            scenario: SCA_STANDALONE_EXPERIMENTAL
           - library: ruby
             app: rack
             scenario: SAMPLING

lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -147,8 +147,6 @@ def add_appsec_tags(processor, context)
             return unless trace && span
 
             span.set_metric(Datadog::AppSec::Ext::TAG_APPSEC_ENABLED, 1)
-            # We add this tag when ASM standalone is enabled to make sure we don't bill APM
-            span.set_metric(Datadog::AppSec::Ext::TAG_APM_ENABLED, 0) if Datadog.configuration.appsec.standalone.enabled
             span.set_tag('_dd.runtime_family', 'ruby')
             span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 

lib/datadog/appsec/ext.rb

@@ -12,7 +12,6 @@ module Ext
       ACTIVE_CONTEXT_KEY = :datadog_appsec_active_context
 
       TAG_APPSEC_ENABLED = '_dd.appsec.enabled'
-      TAG_APM_ENABLED = '_dd.apm.enabled'
       TAG_DISTRIBUTED_APPSEC_EVENT = '_dd.p.appsec'
 
       TELEMETRY_METRICS_NAMESPACE = 'appsec'

lib/datadog/appsec/utils.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'utils/trace_operation'
-
 module Datadog
   module AppSec
     # Utilities for AppSec

lib/datadog/tracing/component.rb

@@ -79,13 +79,7 @@ def build_sampler(settings)
         # so we want to send the minimum amount of traces possible (idealy only traces that contains security events),
         # but for features such as API Security, we need to send at least one trace per minute,
         # to keep the service alive on the backend side.
-        if settings.appsec.standalone.enabled
-          post_sampler = Tracing::Sampling::RuleSampler.new(
-            [Tracing::Sampling::SimpleRule.new(sample_rate: 1.0)],
-            rate_limiter: Datadog::Core::TokenBucket.new(1.0 / 60, 1.0),
-            default_sample_rate: 1.0 / 60
-          )
-        end
+        post_sampler = build_rate_limit_post_sampler(seconds: 60) if settings.appsec.standalone.enabled
 
         # Sampling rules are provided
         if (rules = settings.tracing.sampling.rules)
@@ -197,6 +191,13 @@ def build_tracer_tags(settings)
         end
       end
 
+      def build_rate_limit_post_sampler(seconds:)
+        Tracing::Sampling::RuleSampler.new(
+          rate_limiter: Datadog::Core::TokenBucket.new(1.0 / seconds, 1.0),
+          default_sample_rate: 1.0
+        )
+      end
+
       def build_test_mode_trace_flush(settings)
         # If context flush behavior is provided, use it instead.
         settings.tracing.test_mode.trace_flush || build_trace_flush(settings)

lib/datadog/tracing/contrib/ethon/easy_patch.rb

@@ -110,11 +110,10 @@ def datadog_before_request(continue_from: nil)
 
               datadog_tag_request
 
-              if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(datadog_trace)
-                datadog_trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
-              end
-
-              if datadog_configuration[:distributed_tracing]
+              unless Tracing::Distributed::CircuitBreaker.should_skip_distributed_tracing?(
+                datadog_config: datadog_configuration,
+                trace: datadog_trace
+              )
                 @datadog_original_headers ||= {}
                 Contrib::HTTP.inject(datadog_trace, @datadog_original_headers)
                 self.headers = @datadog_original_headers

lib/datadog/tracing/contrib/excon/middleware.rb

@@ -30,10 +30,12 @@ def request_call(datum)
                 trace = Tracing.active_trace
                 datum[:datadog_span] = span
                 annotate!(span, datum)
-                if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                  trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
+                if Tracing.enabled? && !Tracing::Distributed::CircuitBreaker.should_skip_distributed_tracing?(
+                  datadog_config: @options,
+                  trace: trace
+                )
+                  propagate!(trace, span, datum)
                 end
-                propagate!(trace, span, datum) if distributed_tracing?
 
                 span
               end

lib/datadog/tracing/contrib/faraday/middleware.rb

@@ -29,10 +29,12 @@ def call(env)
 
             Tracing.trace(Ext::SPAN_REQUEST, on_error: request_options[:on_error]) do |span, trace|
               annotate!(span, env, request_options)
-              if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
+              if Tracing.enabled? && !Tracing::Distributed::CircuitBreaker.should_skip_distributed_tracing?(
+                datadog_config: request_options,
+                trace: trace
+              )
+                propagate!(trace, span, env)
               end
-              propagate!(trace, span, env) if request_options[:distributed_tracing] && Tracing.enabled?
               app.call(env).on_complete { |resp| handle_response(span, resp, request_options) }
             end
           end

lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb

@@ -81,7 +81,12 @@ def annotate!(trace, span, keywords, formatter)
               # Set analytics sample rate
               Contrib::Analytics.set_sample_rate(span, analytics_sample_rate) if analytics_enabled?
 
-              GRPC.inject(trace, metadata) if distributed_tracing?
+              unless Tracing::Distributed::CircuitBreaker.should_skip_distributed_tracing?(
+                datadog_config: Datadog.configuration_for(self) || datadog_configuration,
+                trace: trace
+              )
+                GRPC.inject(trace, metadata)
+              end
               Contrib::SpanAttributeSchema.set_peer_service!(span, Ext::PEER_SERVICE_SOURCES)
             rescue StandardError => e
               Datadog.logger.debug("GRPC client trace failed: #{e}")

lib/datadog/tracing/contrib/grpc/distributed/propagation.rb

@@ -3,6 +3,7 @@
 require_relative 'fetcher'
 require_relative '../../../distributed/b3_multi'
 require_relative '../../../distributed/b3_single'
+require_relative '../../../distributed/circuit_breaker'
 require_relative '../../../distributed/datadog'
 require_relative '../../../distributed/none'
 require_relative '../../../distributed/propagation'

lib/datadog/tracing/contrib/http/circuit_breaker.rb

@@ -27,21 +27,6 @@ def internal_request?(request)
             !!(request[Datadog::Core::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION] ||
               request[Datadog::Core::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST])
           end
-
-          def should_skip_distributed_tracing?(client_config)
-            if Datadog.configuration.appsec.standalone.enabled
-              # Skip distributed tracing so that we don't bill distributed traces in case of absence of
-              # upstream ASM event (_dd.p.appsec:1) and no local security event (which sets _dd.p.appsec:1 locally).
-              # If there is an ASM event, we still have to check if distributed tracing is enabled or not
-              return true unless Tracing.active_trace
-
-              return true if Tracing.active_trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1'
-            end
-
-            return !client_config[:distributed_tracing] if client_config && client_config.key?(:distributed_tracing)
-
-            !Datadog.configuration.tracing[:http][:distributed_tracing]
-          end
         end
       end
     end

lib/datadog/tracing/contrib/http/distributed/propagation.rb

@@ -2,6 +2,7 @@
 
 require_relative 'fetcher'
 require_relative '../../../distributed/propagation'
+require_relative '../../../distributed/circuit_breaker'
 require_relative '../../../distributed/b3_multi'
 require_relative '../../../distributed/b3_single'
 require_relative '../../../distributed/datadog'

lib/datadog/tracing/contrib/http/instrumentation.rb

@@ -35,11 +35,11 @@ def request(req, body = nil, &block)
                 span.type = Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
                 span.resource = req.method
 
-                if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                  trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
-                end
-
-                if Tracing.enabled? && !Contrib::HTTP.should_skip_distributed_tracing?(client_config)
+                if Tracing.enabled? && !Tracing::Distributed::CircuitBreaker.should_skip_distributed_tracing?(
+                  client_config: client_config,
+                  datadog_config: Datadog.configuration.tracing[:http],
+                  trace: trace
+                )
                   Contrib::HTTP.inject(trace, req)
                 end
 

lib/datadog/tracing/contrib/httpclient/instrumentation.rb

@@ -30,11 +30,11 @@ def do_get_block(req, proxy, conn, &block)
                   span.service = service_name(host, request_options, client_config)
                   span.type = Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
 
-                  if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                    trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
-                  end
-
-                  if Tracing.enabled? && !should_skip_distributed_tracing?(client_config)
+                  if Tracing.enabled? && !Tracing::Distributed::CircuitBreaker.should_skip_distributed_tracing?(
+                    client_config: client_config,
+                    datadog_config: Datadog.configuration.tracing[:httpclient],
+                    trace: trace
+                  )
                     Contrib::HTTP.inject(trace, req.header)
                   end
 
@@ -123,12 +123,6 @@ def analytics_enabled?(request_options)
               Contrib::Analytics.enabled?(request_options[:analytics_enabled])
             end
 
-            def should_skip_distributed_tracing?(client_config)
-              return !client_config[:distributed_tracing] if client_config && client_config.key?(:distributed_tracing)
-
-              !Datadog.configuration.tracing[:httpclient][:distributed_tracing]
-            end
-
             def set_analytics_sample_rate(span, request_options)
               return unless analytics_enabled?(request_options)
 

lib/datadog/tracing/contrib/httprb/instrumentation.rb

@@ -30,12 +30,14 @@ def perform(req, options)
                   span.service = service_name(host, request_options, client_config)
                   span.type = Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
 
-                  if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                    trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
+                  if Tracing.enabled? && !Tracing::Distributed::CircuitBreaker.should_skip_distributed_tracing?(
+                    client_config: client_config,
+                    datadog_config: Datadog.configuration.tracing[:httprb],
+                    trace: trace
+                  )
+                    Contrib::HTTP.inject(trace, req)
                   end
 
-                  Contrib::HTTP.inject(trace, req) if Tracing.enabled? && !should_skip_distributed_tracing?(client_config)
-
                   # Add additional request specific tags to the span.
                   annotate_span_with_request!(span, req, request_options)
                 rescue StandardError => e
@@ -135,12 +137,6 @@ def logger
               Datadog.logger
             end
 
-            def should_skip_distributed_tracing?(client_config)
-              return !client_config[:distributed_tracing] if client_config && client_config.key?(:distributed_tracing)
-
-              !Datadog.configuration.tracing[:httprb][:distributed_tracing]
-            end
-
             def set_analytics_sample_rate(span, request_options)
               return unless analytics_enabled?(request_options)
 

lib/datadog/tracing/contrib/rest_client/request_patch.rb

@@ -25,10 +25,12 @@ def execute(&block)
               return super(&block) unless Tracing.enabled?
 
               datadog_trace_request(uri) do |_span, trace|
-                if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                  trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
+                unless Tracing::Distributed::CircuitBreaker.should_skip_distributed_tracing?(
+                  datadog_config: datadog_configuration,
+                  trace: trace
+                )
+                  Contrib::HTTP.inject(trace, processed_headers)
                 end
-                Contrib::HTTP.inject(trace, processed_headers) if datadog_configuration[:distributed_tracing]
 
                 super(&block)
               end

lib/datadog/tracing/contrib/sidekiq/client_tracer.rb

@@ -24,7 +24,12 @@ def call(worker_class, job, queue, redis_pool)
             resource = job_resource(job)
 
             Datadog::Tracing.trace(Ext::SPAN_PUSH, service: @sidekiq_service) do |span, trace_op|
-              Sidekiq.inject(trace_op, job) if configuration[:distributed_tracing]
+              unless Tracing::Distributed::CircuitBreaker.should_skip_distributed_tracing?(
+                datadog_config: configuration,
+                trace: trace_op
+              )
+                Sidekiq.inject(trace_op, job)
+              end
 
               span.resource = resource
 

lib/datadog/tracing/contrib/sidekiq/distributed/propagation.rb

@@ -2,6 +2,7 @@
 
 require_relative '../../../distributed/fetcher'
 require_relative '../../../distributed/propagation'
+require_relative '../../../distributed/circuit_breaker'
 require_relative '../../../distributed/b3_multi'
 require_relative '../../../distributed/b3_single'
 require_relative '../../../distributed/datadog'

lib/datadog/tracing/distributed/circuit_breaker.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Distributed
+      # Helper method to decide when to skip distributed tracing
+      module CircuitBreaker
+        module_function
+
+        # Skips distributed tracing if disabled for this instrumentation
+        # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+        def should_skip_distributed_tracing?(client_config: nil, datadog_config: nil, trace: nil)
+          if ::Datadog.configuration.appsec.standalone.enabled &&
+              (trace.nil? || trace.get_tag(::Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+            return true
+          end
+
+          return !client_config[:distributed_tracing] if client_config && client_config.key?(:distributed_tracing)
+          return !datadog_config[:distributed_tracing] if datadog_config
+
+          false
+        end
+      end
+    end
+  end
+end

lib/datadog/tracing/metadata/ext.rb

@@ -31,6 +31,8 @@ module Ext
         # See Datadog-internal "RFC: Identifying which spans have profiling enabled " for details
         TAG_PROFILING_ENABLED = '_dd.profiling.enabled'
 
+        TAG_APM_ENABLED = '_dd.apm.enabled'
+
         # Defines constants for trace analytics
         # @public_api
         module Analytics

lib/datadog/tracing/trace_operation.rb

@@ -71,13 +71,13 @@ def initialize(
         sampling_priority: nil,
         service: nil,
         profiling_enabled: nil,
+        apm_tracing_disabled: nil,
         tags: nil,
         metrics: nil,
         trace_state: nil,
         trace_state_unknown_fields: nil,
         remote_parent: false,
         tracer: nil
-
       )
         # Attributes
         @id = id || Tracing::Utils::TraceId.next_id
@@ -98,6 +98,7 @@ def initialize(
         @sampling_priority = sampling_priority
         @service = service
         @profiling_enabled = profiling_enabled
+        @apm_tracing_disabled = apm_tracing_disabled
         @trace_state = trace_state
         @trace_state_unknown_fields = trace_state_unknown_fields
         @tracer = tracer
@@ -510,6 +511,7 @@ def build_trace(spans, partial = false)
           metrics: metrics,
           root_span_id: !partial ? root_span && root_span.id : nil,
           profiling_enabled: @profiling_enabled,
+          apm_tracing_disabled: @apm_tracing_disabled
         )
       end