Fix APM Disablement V1 (ASM Standalone)

.github/forced-tests-list.json

@@ -1,3 +1,5 @@
 {
-
-}
+  "SCA_STANDALONE": [
+    "tests/appsec/test_asm_standalone.py::Test_SCAStandalone_Telemetry"
+  ]
+}

.github/workflows/system-tests.yml

@@ -320,6 +320,9 @@ jobs:
           - library: ruby
             app: rails70
             scenario: DEBUGGER_PII_REDACTION
+          - library: ruby
+            app: rails70
+            scenario: SCA_STANDALONE
           - library: ruby
             app: rack
             scenario: SAMPLING

lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -147,8 +147,6 @@ def add_appsec_tags(processor, context)
             return unless trace && span
 
             span.set_metric(Datadog::AppSec::Ext::TAG_APPSEC_ENABLED, 1)
-            # We add this tag when ASM standalone is enabled to make sure we don't bill APM
-            span.set_metric(Datadog::AppSec::Ext::TAG_APM_ENABLED, 0) if Datadog.configuration.appsec.standalone.enabled
             span.set_tag('_dd.runtime_family', 'ruby')
             span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 

lib/datadog/appsec/ext.rb

@@ -12,7 +12,6 @@ module Ext
       ACTIVE_CONTEXT_KEY = :datadog_appsec_active_context
 
       TAG_APPSEC_ENABLED = '_dd.appsec.enabled'
-      TAG_APM_ENABLED = '_dd.apm.enabled'
       TAG_DISTRIBUTED_APPSEC_EVENT = '_dd.p.appsec'
 
       TELEMETRY_METRICS_NAMESPACE = 'appsec'

lib/datadog/appsec/utils.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'utils/trace_operation'
-
 module Datadog
   module AppSec
     # Utilities for AppSec

lib/datadog/tracing/component.rb

@@ -79,13 +79,7 @@ def build_sampler(settings)
         # so we want to send the minimum amount of traces possible (idealy only traces that contains security events),
         # but for features such as API Security, we need to send at least one trace per minute,
         # to keep the service alive on the backend side.
-        if settings.appsec.standalone.enabled
-          post_sampler = Tracing::Sampling::RuleSampler.new(
-            [Tracing::Sampling::SimpleRule.new(sample_rate: 1.0)],
-            rate_limiter: Datadog::Core::TokenBucket.new(1.0 / 60, 1.0),
-            default_sample_rate: 1.0 / 60
-          )
-        end
+        post_sampler = build_rate_limit_post_sampler(seconds: 60) if settings.appsec.standalone.enabled
 
         # Sampling rules are provided
         if (rules = settings.tracing.sampling.rules)
@@ -197,6 +191,13 @@ def build_tracer_tags(settings)
         end
       end
 
+      def build_rate_limit_post_sampler(seconds:)
+        Tracing::Sampling::RuleSampler.new(
+          rate_limiter: Datadog::Core::TokenBucket.new(1.0 / seconds, 1.0),
+          default_sample_rate: 1.0
+        )
+      end
+
       def build_test_mode_trace_flush(settings)
         # If context flush behavior is provided, use it instead.
         settings.tracing.test_mode.trace_flush || build_trace_flush(settings)

lib/datadog/tracing/contrib/ethon/easy_patch.rb

@@ -110,11 +110,7 @@ def datadog_before_request(continue_from: nil)
 
               datadog_tag_request
 
-              if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(datadog_trace)
-                datadog_trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
-              end
-
-              if datadog_configuration[:distributed_tracing]
+              unless should_skip_distributed_tracing?(datadog_trace)
                 @datadog_original_headers ||= {}
                 Contrib::HTTP.inject(datadog_trace, @datadog_original_headers)
                 self.headers = @datadog_original_headers
@@ -219,6 +215,17 @@ def parse_response_headers
                 header.size != 2 ? nil : header
               end.compact.to_h
             end
+
+            # Skips distributed tracing if disabled for this instrumentation
+            # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+            def should_skip_distributed_tracing?(trace)
+              if Datadog.configuration.appsec.standalone.enabled &&
+                  (trace.nil? || trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+                return true
+              end
+
+              !datadog_configuration[:distributed_tracing]
+            end
           end
         end
       end

lib/datadog/tracing/contrib/excon/middleware.rb

@@ -30,10 +30,7 @@ def request_call(datum)
                 trace = Tracing.active_trace
                 datum[:datadog_span] = span
                 annotate!(span, datum)
-                if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                  trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
-                end
-                propagate!(trace, span, datum) if distributed_tracing?
+                propagate!(trace, span, datum) unless should_skip_distributed_tracing?(trace)
 
                 span
               end
@@ -192,6 +189,17 @@ def build_request_options!(datum)
           def datadog_configuration(host = :default)
             Datadog.configuration.tracing[:excon, host]
           end
+
+          # Skips distributed tracing if disabled for this instrumentation
+          # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+          def should_skip_distributed_tracing?(trace)
+            if Datadog.configuration.appsec.standalone.enabled &&
+                (trace.nil? || trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+              return true
+            end
+
+            !distributed_tracing?
+          end
         end
       end
     end

lib/datadog/tracing/contrib/faraday/middleware.rb

@@ -29,10 +29,7 @@ def call(env)
 
             Tracing.trace(Ext::SPAN_REQUEST, on_error: request_options[:on_error]) do |span, trace|
               annotate!(span, env, request_options)
-              if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
-              end
-              propagate!(trace, span, env) if request_options[:distributed_tracing] && Tracing.enabled?
+              propagate!(trace, span, env) if Tracing.enabled? && !should_skip_distributed_tracing?(request_options, trace)
               app.call(env).on_complete { |resp| handle_response(span, resp, request_options) }
             end
           end
@@ -117,6 +114,17 @@ def build_request_options!(env)
           def datadog_configuration(host = :default)
             Datadog.configuration.tracing[:faraday, host]
           end
+
+          # Skips distributed tracing if disabled for this instrumentation
+          # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+          def should_skip_distributed_tracing?(client_config, trace)
+            if Datadog.configuration.appsec.standalone.enabled &&
+                (trace.nil? || trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+              return true
+            end
+
+            !client_config[:distributed_tracing]
+          end
         end
       end
     end

lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb

@@ -81,7 +81,7 @@ def annotate!(trace, span, keywords, formatter)
               # Set analytics sample rate
               Contrib::Analytics.set_sample_rate(span, analytics_sample_rate) if analytics_enabled?
 
-              GRPC.inject(trace, metadata) if distributed_tracing?
+              GRPC.inject(trace, metadata) unless should_skip_distributed_tracing?(trace)
               Contrib::SpanAttributeSchema.set_peer_service!(span, Ext::PEER_SERVICE_SOURCES)
             rescue StandardError => e
               Datadog.logger.debug("GRPC client trace failed: #{e}")
@@ -109,6 +109,17 @@ def find_host_port(call)
               Datadog.logger.debug { "Could not parse host:port from #{call}: #{e}" }
               nil
             end
+
+            # Skips distributed tracing if disabled for this instrumentation
+            # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+            def should_skip_distributed_tracing?(trace)
+              if Datadog.configuration.appsec.standalone.enabled &&
+                  (trace.nil? || trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+                return true
+              end
+
+              !distributed_tracing?
+            end
           end
         end
       end

lib/datadog/tracing/contrib/http/circuit_breaker.rb

@@ -28,14 +28,12 @@ def internal_request?(request)
               request[Datadog::Core::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST])
           end
 
-          def should_skip_distributed_tracing?(client_config)
-            if Datadog.configuration.appsec.standalone.enabled
-              # Skip distributed tracing so that we don't bill distributed traces in case of absence of
-              # upstream ASM event (_dd.p.appsec:1) and no local security event (which sets _dd.p.appsec:1 locally).
-              # If there is an ASM event, we still have to check if distributed tracing is enabled or not
-              return true unless Tracing.active_trace
-
-              return true if Tracing.active_trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1'
+          # Skips distributed tracing if disabled for this instrumentation
+          # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+          def should_skip_distributed_tracing?(client_config, trace)
+            if Datadog.configuration.appsec.standalone.enabled &&
+                (trace.nil? || trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+              return true
             end
 
             return !client_config[:distributed_tracing] if client_config && client_config.key?(:distributed_tracing)

lib/datadog/tracing/contrib/http/instrumentation.rb

@@ -35,11 +35,7 @@ def request(req, body = nil, &block)
                 span.type = Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
                 span.resource = req.method
 
-                if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                  trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
-                end
-
-                if Tracing.enabled? && !Contrib::HTTP.should_skip_distributed_tracing?(client_config)
+                if Tracing.enabled? && !Contrib::HTTP.should_skip_distributed_tracing?(client_config, trace)
                   Contrib::HTTP.inject(trace, req)
                 end
 

lib/datadog/tracing/contrib/httpclient/instrumentation.rb

@@ -30,11 +30,7 @@ def do_get_block(req, proxy, conn, &block)
                   span.service = service_name(host, request_options, client_config)
                   span.type = Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
 
-                  if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                    trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
-                  end
-
-                  if Tracing.enabled? && !should_skip_distributed_tracing?(client_config)
+                  if Tracing.enabled? && !should_skip_distributed_tracing?(client_config, trace)
                     Contrib::HTTP.inject(trace, req.header)
                   end
 
@@ -123,7 +119,14 @@ def analytics_enabled?(request_options)
               Contrib::Analytics.enabled?(request_options[:analytics_enabled])
             end
 
-            def should_skip_distributed_tracing?(client_config)
+            # Skips distributed tracing if disabled for this instrumentation
+            # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+            def should_skip_distributed_tracing?(client_config, trace)
+              if Datadog.configuration.appsec.standalone.enabled &&
+                  (trace.nil? || trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+                return true
+              end
+
               return !client_config[:distributed_tracing] if client_config && client_config.key?(:distributed_tracing)
 
               !Datadog.configuration.tracing[:httpclient][:distributed_tracing]

lib/datadog/tracing/contrib/httprb/instrumentation.rb

@@ -30,12 +30,10 @@ def perform(req, options)
                   span.service = service_name(host, request_options, client_config)
                   span.type = Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
 
-                  if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                    trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
+                  if Tracing.enabled? && !should_skip_distributed_tracing?(client_config, trace)
+                    Contrib::HTTP.inject(trace, req)
                   end
 
-                  Contrib::HTTP.inject(trace, req) if Tracing.enabled? && !should_skip_distributed_tracing?(client_config)
-
                   # Add additional request specific tags to the span.
                   annotate_span_with_request!(span, req, request_options)
                 rescue StandardError => e
@@ -135,7 +133,14 @@ def logger
               Datadog.logger
             end
 
-            def should_skip_distributed_tracing?(client_config)
+            # Skips distributed tracing if disabled for this instrumentation
+            # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+            def should_skip_distributed_tracing?(client_config, trace)
+              if Datadog.configuration.appsec.standalone.enabled &&
+                  (trace.nil? || trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+                return true
+              end
+
               return !client_config[:distributed_tracing] if client_config && client_config.key?(:distributed_tracing)
 
               !Datadog.configuration.tracing[:httprb][:distributed_tracing]

lib/datadog/tracing/contrib/rest_client/request_patch.rb

@@ -25,10 +25,7 @@ def execute(&block)
               return super(&block) unless Tracing.enabled?
 
               datadog_trace_request(uri) do |_span, trace|
-                if Datadog::AppSec::Utils::TraceOperation.appsec_standalone_reject?(trace)
-                  trace.sampling_priority = Tracing::Sampling::Ext::Priority::AUTO_REJECT
-                end
-                Contrib::HTTP.inject(trace, processed_headers) if datadog_configuration[:distributed_tracing]
+                Contrib::HTTP.inject(trace, processed_headers) unless should_skip_distributed_tracing?(trace)
 
                 super(&block)
               end
@@ -124,6 +121,17 @@ def analytics_enabled?
             def analytics_sample_rate
               datadog_configuration[:analytics_sample_rate]
             end
+
+            # Skips distributed tracing if disabled for this instrumentation
+            # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+            def should_skip_distributed_tracing?(trace)
+              if Datadog.configuration.appsec.standalone.enabled &&
+                  (trace.nil? || trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+                return true
+              end
+
+              !datadog_configuration[:distributed_tracing]
+            end
           end
         end
       end

lib/datadog/tracing/contrib/sidekiq/client_tracer.rb

@@ -24,7 +24,7 @@ def call(worker_class, job, queue, redis_pool)
             resource = job_resource(job)
 
             Datadog::Tracing.trace(Ext::SPAN_PUSH, service: @sidekiq_service) do |span, trace_op|
-              Sidekiq.inject(trace_op, job) if configuration[:distributed_tracing]
+              Sidekiq.inject(trace_op, job) unless should_skip_distributed_tracing?(trace_op)
 
               span.resource = resource
 
@@ -55,6 +55,17 @@ def call(worker_class, job, queue, redis_pool)
           def configuration
             Datadog.configuration.tracing[:sidekiq]
           end
+
+          # Skips distributed tracing if disabled for this instrumentation
+          # or if APM is disabled unless there is an AppSec event (from upstream distributed trace or local)
+          def should_skip_distributed_tracing?(trace)
+            if Datadog.configuration.appsec.standalone.enabled &&
+                (trace.nil? || trace.get_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT) != '1')
+              return true
+            end
+
+            !configuration[:distributed_tracing]
+          end
         end
       end
     end

lib/datadog/tracing/metadata/ext.rb

@@ -31,6 +31,8 @@ module Ext
         # See Datadog-internal "RFC: Identifying which spans have profiling enabled " for details
         TAG_PROFILING_ENABLED = '_dd.profiling.enabled'
 
+        TAG_APM_ENABLED = '_dd.apm.enabled'
+
         # Defines constants for trace analytics
         # @public_api
         module Analytics

lib/datadog/tracing/trace_operation.rb

@@ -71,13 +71,13 @@ def initialize(
         sampling_priority: nil,
         service: nil,
         profiling_enabled: nil,
+        apm_tracing_disabled: nil,
         tags: nil,
         metrics: nil,
         trace_state: nil,
         trace_state_unknown_fields: nil,
         remote_parent: false,
         tracer: nil
-
       )
         # Attributes
         @id = id || Tracing::Utils::TraceId.next_id
@@ -98,6 +98,7 @@ def initialize(
         @sampling_priority = sampling_priority
         @service = service
         @profiling_enabled = profiling_enabled
+        @apm_tracing_disabled = apm_tracing_disabled
         @trace_state = trace_state
         @trace_state_unknown_fields = trace_state_unknown_fields
         @tracer = tracer
@@ -510,6 +511,7 @@ def build_trace(spans, partial = false)
           metrics: metrics,
           root_span_id: !partial ? root_span && root_span.id : nil,
           profiling_enabled: @profiling_enabled,
+          apm_tracing_disabled: @apm_tracing_disabled
         )
       end