diff --git a/lib/datadog/appsec.rb b/lib/datadog/appsec.rb index 968b9c6a1fb..fd67d985066 100644 --- a/lib/datadog/appsec.rb +++ b/lib/datadog/appsec.rb @@ -5,6 +5,7 @@ require_relative 'appsec/context' require_relative 'appsec/ext' require_relative 'appsec/utils' +require_relative 'appsec/stack_trace' module Datadog # Namespace for Datadog AppSec instrumentation diff --git a/lib/datadog/appsec/contrib/active_record/instrumentation.rb b/lib/datadog/appsec/contrib/active_record/instrumentation.rb index 41a8e80523c..d4b27410aa7 100644 --- a/lib/datadog/appsec/contrib/active_record/instrumentation.rb +++ b/lib/datadog/appsec/contrib/active_record/instrumentation.rb @@ -36,6 +36,11 @@ def detect_sql_injection(sql, adapter_name) actions: result.actions } context.waf_runner.events << event + + # Include the stack trace if waf result includes it + if AppSec::StackTrace.include_stack_trace?(result) + AppSec::StackTrace.add_stack_trace(context, result) + end end end diff --git a/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb b/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb index 025b6e10359..5ef9149a1ef 100644 --- a/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb +++ b/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb @@ -39,6 +39,11 @@ def watch_multiplex(gateway = Instrumentation.gateway) Datadog::AppSec::Event.tag_and_keep!(context, result) context.waf_runner.events << event + + # Include the stack trace if waf result includes it + if AppSec::StackTrace.include_stack_trace?(result) + AppSec::StackTrace.add_stack_trace(context, result) + end end block = GraphQL::Reactive::Multiplex.publish(engine, gateway_multiplex) diff --git a/lib/datadog/appsec/contrib/rack/gateway/watcher.rb b/lib/datadog/appsec/contrib/rack/gateway/watcher.rb index cab7ae3128e..75c2dec6efe 100644 --- a/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +++ b/lib/datadog/appsec/contrib/rack/gateway/watcher.rb @@ -44,6 +44,11 @@ def watch_request(gateway = Instrumentation.gateway) context.trace.keep! if context.trace Datadog::AppSec::Event.tag_and_keep!(context, result) context.waf_runner.events << event + + # Include the stack trace if waf result includes it + if AppSec::StackTrace.include_stack_trace?(result) + AppSec::StackTrace.add_stack_trace(context, result) + end end end @@ -75,6 +80,11 @@ def watch_response(gateway = Instrumentation.gateway) context.trace.keep! if context.trace Datadog::AppSec::Event.tag_and_keep!(context, result) context.waf_runner.events << event + + # Include the stack trace if waf result includes it + if AppSec::StackTrace.include_stack_trace?(result) + AppSec::StackTrace.add_stack_trace(context, result) + end end end @@ -106,6 +116,11 @@ def watch_request_body(gateway = Instrumentation.gateway) context.trace.keep! if context.trace Datadog::AppSec::Event.tag_and_keep!(context, result) context.waf_runner.events << event + + # Include the stack trace if waf result includes it + if AppSec::StackTrace.include_stack_trace?(result) + AppSec::StackTrace.add_stack_trace(context, result) + end end end diff --git a/lib/datadog/appsec/contrib/rails/gateway/watcher.rb b/lib/datadog/appsec/contrib/rails/gateway/watcher.rb index 4ae52c73333..56645353300 100644 --- a/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +++ b/lib/datadog/appsec/contrib/rails/gateway/watcher.rb @@ -40,6 +40,11 @@ def watch_request_action(gateway = Instrumentation.gateway) context.trace.keep! if context.trace Datadog::AppSec::Event.tag_and_keep!(context, result) context.waf_runner.events << event + + # Include the stack trace if waf result includes it + if AppSec::StackTrace.include_stack_trace?(result) + AppSec::StackTrace.add_stack_trace(context, result) + end end end diff --git a/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb b/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb index a9ca382b904..8ea87a3d834 100644 --- a/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +++ b/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb @@ -42,6 +42,11 @@ def watch_request_dispatch(gateway = Instrumentation.gateway) context.trace.keep! if context.trace Datadog::AppSec::Event.tag_and_keep!(context, result) context.waf_runner.events << event + + # Include the stack trace if waf result includes it + if AppSec::StackTrace.include_stack_trace?(result) + AppSec::StackTrace.add_stack_trace(context, result) + end end end @@ -73,6 +78,11 @@ def watch_request_routed(gateway = Instrumentation.gateway) context.trace.keep! if context.trace Datadog::AppSec::Event.tag_and_keep!(context, result) context.waf_runner.events << event + + # Include the stack trace if waf result includes it + if AppSec::StackTrace.include_stack_trace?(result) + AppSec::StackTrace.add_stack_trace(context, result) + end end end diff --git a/lib/datadog/appsec/ext.rb b/lib/datadog/appsec/ext.rb index 2c6e6f5fae3..0e660841500 100644 --- a/lib/datadog/appsec/ext.rb +++ b/lib/datadog/appsec/ext.rb @@ -7,10 +7,12 @@ module Ext INTERRUPT = :datadog_appsec_interrupt CONTEXT_KEY = 'datadog.appsec.context' ACTIVE_CONTEXT_KEY = :datadog_appsec_active_context + EXPLOIT_PREVENTION_EVENT_CATEGORY = 'exploit' TAG_APPSEC_ENABLED = '_dd.appsec.enabled' TAG_APM_ENABLED = '_dd.apm.enabled' TAG_DISTRIBUTED_APPSEC_EVENT = '_dd.p.appsec' + TAG_STACK_TRACE = '_dd.stack' end end end diff --git a/lib/datadog/appsec/monitor/gateway/watcher.rb b/lib/datadog/appsec/monitor/gateway/watcher.rb index f7d29bf5601..6d891503192 100644 --- a/lib/datadog/appsec/monitor/gateway/watcher.rb +++ b/lib/datadog/appsec/monitor/gateway/watcher.rb @@ -38,6 +38,11 @@ def watch_user_id(gateway = Instrumentation.gateway) context.trace.keep! if context.trace Datadog::AppSec::Event.tag_and_keep!(context, result) context.waf_runner.events << event + + # Include the stack trace if waf result includes it + if AppSec::StackTrace.include_stack_trace?(result) + AppSec::StackTrace.add_stack_trace(context, result) + end end end diff --git a/lib/datadog/appsec/stack_trace.rb b/lib/datadog/appsec/stack_trace.rb index 7543b91dbd5..327874031bb 100644 --- a/lib/datadog/appsec/stack_trace.rb +++ b/lib/datadog/appsec/stack_trace.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'msgpack' + module Datadog module AppSec # Stack trace @@ -33,6 +35,25 @@ def to_h } end + def to_msgpack(packer = nil) + packer ||= MessagePack::Packer.new + + packer.write_map_header(4) + packer.write('language') + packer.write('ruby') + packer.write('id') + packer.write(@id) + packer.write('message') + packer.write(@message) + packer.write('frames') + packer.write(@frames) + packer + end + + def pretty_print(q) + q.text to_s + end + class << self # Create a stack trace from the result of WAF execution def from_waf_result(waf_result) @@ -45,7 +66,29 @@ def from_waf_result(waf_result) # Whether to include the stack trace in the event def include_stack_trace?(waf_result) - Datadog.configuration.appsec.stack_trace.enabled && waf_result.actions.key?('generate_stack') + Datadog.configuration.appsec.stack_trace.enabled && waf_result.actions.key?('generate_stack') + end + + # Add the stack trace to the trace + def add_stack_trace(scope, result) + # We use methods defined in Tracing::Metadata::Tagging, + # which means we can work on both the trace and the service entry span + service_entry_operation = scope.trace || scope.span + + unless service_entry_operation + Datadog.logger.debug { "Cannot find trace or service entry span to add stack trace" } + return + end + + stack_trace = from_waf_result(result) + + service_entry_operation.modify_meta_struct_tag(AppSec::Ext::TAG_STACK_TRACE) do |stack_traces| + # _dd.stack is a hash of stack traces (Hash>) + stack_traces ||= {} + stack_traces[AppSec::Ext::EXPLOIT_PREVENTION_EVENT_CATEGORY] ||= [] + stack_traces[AppSec::Ext::EXPLOIT_PREVENTION_EVENT_CATEGORY] << stack_trace + stack_traces + end end end @@ -70,7 +113,24 @@ def to_h function: @function } end + + def to_msgpack(packer = nil) + packer ||= MessagePack::Packer.new + + packer.write_map_header(5) + packer.write('id') + packer.write(@id) + packer.write('text') + packer.write(@text) + packer.write('file') + packer.write(@file) + packer.write('line') + packer.write(@line) + packer.write('function') + packer.write(@function) + packer + end end end end -end \ No newline at end of file +end diff --git a/lib/datadog/tracing/metadata/tagging.rb b/lib/datadog/tracing/metadata/tagging.rb index ef89cd769d2..f6eaa18bdd6 100644 --- a/lib/datadog/tracing/metadata/tagging.rb +++ b/lib/datadog/tracing/metadata/tagging.rb @@ -65,6 +65,10 @@ def set_tags(hash) hash.each { |k, v| set_tag(k, v) } end + def set_meta_struct_tags(hash) + hash.each { |k, v| set_meta_struct_tag(k, v) } + end + # Returns true if the provided `tag` was set to a non-nil value. # False otherwise. # @@ -111,29 +115,44 @@ def clear_metric(key) end # Getter for meta_struct - def get_struct_tag(key) + def get_meta_struct_tag(key) meta_struct[key] end # Setter for meta_struct, key must be a string, value can be anything - # Keys can be duplicated in meta_struct and meta/metrics - def set_struct_tag(key, value) - meta_struct[key] = value + # Keys can be duplicated in meta_struct and meta/metrics but because each tag have a different structure + # we must create a condition for each. Default behaviour erases the previous value. + def set_meta_struct_tag(key, value) + if key == Datadog::AppSec::Ext::TAG_STACK_TRACE && meta_struct.key?(key) + value.each do |event_type, stack_traces| + meta_struct[key][event_type] ||= [] + meta_struct[key][event_type].concat(stack_traces) + end + else + meta_struct[key] = value + end rescue StandardError => e Datadog.logger.debug("Unable to set the tag #{key} in meta_struct, ignoring it. Caused by: #{e}") end + # Modifies the content of a tag in meta_struct + def modify_meta_struct_tag(key) + meta_struct[key] = yield(meta_struct[key]) + rescue StandardError => e + Datadog.logger.debug("Unable to modify the tag #{key} in meta_struct, ignoring it. Caused by: #{e}") + end + # Returns true if the provided `tag` was set to a non-nil value. # False otherwise. # # @param [String] tag the tag or metric to check for presence # @return [Boolean] if the tag is present and not nil - def has_struct_tag?(tag) # rubocop:disable Naming/PredicateName - !get_struct_tag(tag).nil? # nil is considered not present, thus we can't use `Hash#has_key?` + def has_meta_struct_tag?(tag) # rubocop:disable Naming/PredicateName + !get_meta_struct_tag(tag).nil? # nil is considered not present, thus we can't use `Hash#has_key?` end # This method removes a tag for the given key. - def clear_struct_tag(key) + def clear_meta_struct_tag(key) meta_struct.delete(key) end diff --git a/lib/datadog/tracing/span.rb b/lib/datadog/tracing/span.rb index a569dd09896..7938393622b 100644 --- a/lib/datadog/tracing/span.rb +++ b/lib/datadog/tracing/span.rb @@ -142,6 +142,7 @@ def to_hash error: @status, meta: @meta, metrics: @metrics, + meta_struct: @meta_struct, name: @name, parent_id: @parent_id, resource: @resource, @@ -185,12 +186,16 @@ def pretty_print(q) q.text "#{key} => #{value}" end end - q.group(2, 'Metrics: [', ']') do + q.group(2, 'Metrics: [', "]\n") do q.breakable q.seplist @metrics.each do |key, value| q.text "#{key} => #{value}" end end + q.group(2, 'Meta-Struct: [', ']') do + q.breakable + q.pp meta_struct + end end end diff --git a/lib/datadog/tracing/span_operation.rb b/lib/datadog/tracing/span_operation.rb index 52a8fc469af..cb5fde636e1 100644 --- a/lib/datadog/tracing/span_operation.rb +++ b/lib/datadog/tracing/span_operation.rb @@ -285,6 +285,7 @@ def to_hash id: @id, meta: meta, metrics: metrics, + meta_struct: meta_struct, name: @name, parent_id: @parent_id, resource: @resource, @@ -324,12 +325,16 @@ def pretty_print(q) q.text "#{key} => #{value}" end end - q.group(2, 'Metrics: [', ']') do + q.group(2, 'Metrics: [', "]\n") do q.breakable q.seplist metrics.each do |key, value| q.text "#{key} => #{value}" end end + q.group(2, 'Meta-Struct: [', ']') do + q.breakable + q.pp meta_struct + end end end @@ -452,6 +457,7 @@ def build_span id: @id, meta: Core::Utils::SafeDup.frozen_or_dup(meta), metrics: Core::Utils::SafeDup.frozen_or_dup(metrics), + meta_struct: Core::Utils::SafeDup.frozen_or_dup(meta_struct), parent_id: @parent_id, resource: @resource, service: @service, diff --git a/lib/datadog/tracing/trace_operation.rb b/lib/datadog/tracing/trace_operation.rb index eb85182af75..e6a89e53bdf 100644 --- a/lib/datadog/tracing/trace_operation.rb +++ b/lib/datadog/tracing/trace_operation.rb @@ -73,6 +73,7 @@ def initialize( profiling_enabled: nil, tags: nil, metrics: nil, + meta_struct: nil, trace_state: nil, trace_state_unknown_fields: nil, remote_parent: false, @@ -105,6 +106,7 @@ def initialize( # Generic tags set_tags(tags) if tags set_tags(metrics) if metrics + set_meta_struct_tags(meta_struct) if meta_struct # State @root_span = nil @@ -369,6 +371,7 @@ def fork_clone trace_state_unknown_fields: (@trace_state_unknown_fields && @trace_state_unknown_fields.dup), tags: meta.dup, metrics: metrics.dup, + meta_struct: meta_struct.dup, remote_parent: @remote_parent ) end @@ -508,6 +511,7 @@ def build_trace(spans, partial = false) service: service, tags: meta, metrics: metrics, + meta_struct: meta_struct, root_span_id: !partial ? root_span && root_span.id : nil, profiling_enabled: @profiling_enabled, ) diff --git a/lib/datadog/tracing/trace_segment.rb b/lib/datadog/tracing/trace_segment.rb index 6d0903d9a9d..a6caabecbf9 100644 --- a/lib/datadog/tracing/trace_segment.rb +++ b/lib/datadog/tracing/trace_segment.rb @@ -58,6 +58,7 @@ def initialize( service: nil, tags: nil, metrics: nil, + meta_struct: nil, profiling_enabled: nil ) @id = id @@ -68,6 +69,7 @@ def initialize( # The caller is expected to have done that @meta = (tags && tags.dup) || {} @metrics = (metrics && metrics.dup) || {} + @meta_struct = (meta_struct && meta_struct.dup) || {} # Set well-known tags, defaulting to getting the values from tags @agent_sample_rate = agent_sample_rate || agent_sample_rate_tag @@ -146,7 +148,8 @@ def high_order_tid attr_reader \ :root_span_id, :meta, - :metrics + :metrics, + :meta_struct private diff --git a/lib/datadog/tracing/transport/serializable_trace.rb b/lib/datadog/tracing/transport/serializable_trace.rb index 380f8623954..48ba1b73286 100644 --- a/lib/datadog/tracing/transport/serializable_trace.rb +++ b/lib/datadog/tracing/transport/serializable_trace.rb @@ -65,7 +65,7 @@ def initialize(span, native_events_supported) def to_msgpack(packer = nil) packer ||= MessagePack::Packer.new - number_of_elements_to_write = 11 + number_of_elements_to_write = 12 number_of_elements_to_write += 1 if span.events.any? && @native_events_supported @@ -113,6 +113,9 @@ def to_msgpack(packer = nil) packer.write(span.meta) packer.write('metrics') packer.write(span.metrics) + packer.write('meta_struct') + # We encapsulate the resulting msgpack in a binary msgpack + packer.write(span.meta_struct.transform_values(&:to_msgpack)) packer.write('span_links') packer.write(span.links.map(&:to_hash)) packer.write('error') diff --git a/lib/datadog/tracing/transport/trace_formatter.rb b/lib/datadog/tracing/transport/trace_formatter.rb index 2140ccebdd0..ca05fbf1aa9 100644 --- a/lib/datadog/tracing/transport/trace_formatter.rb +++ b/lib/datadog/tracing/transport/trace_formatter.rb @@ -43,6 +43,7 @@ def format! # Apply generic trace tags. Any more specific value will be overridden # by the subsequent calls below. set_trace_tags! + set_meta_struct! set_resource! @@ -89,6 +90,12 @@ def set_trace_tags! root_span.set_tags(trace.send(:metrics)) end + def set_meta_struct! + return if partial? + + root_span.set_meta_struct_tags(trace.send(:meta_struct)) + end + def tag_agent_sample_rate! return unless trace.agent_sample_rate diff --git a/lib/datadog/tracing/transport/traces.rb b/lib/datadog/tracing/transport/traces.rb index 644f80fe94c..43cb7ac0b0b 100644 --- a/lib/datadog/tracing/transport/traces.rb +++ b/lib/datadog/tracing/transport/traces.rb @@ -125,6 +125,7 @@ def initialize(apis, default_api) end def send_traces(traces) + # object that extends Datadog::Core::Encoding::Encoder (MsgpackEncoder or JSONEncoder) encoder = current_api.encoder chunker = Datadog::Tracing::Transport::Traces::Chunker.new(encoder) diff --git a/sig/datadog/appsec/ext.rbs b/sig/datadog/appsec/ext.rbs index 8c4b59fb575..89641aa8f77 100644 --- a/sig/datadog/appsec/ext.rbs +++ b/sig/datadog/appsec/ext.rbs @@ -5,10 +5,12 @@ module Datadog INTERRUPT: ::Symbol CONTEXT_KEY: ::String ACTIVE_CONTEXT_KEY: ::Symbol + EXPLOIT_PREVENTION_EVENT_CATEGORY: ::String TAG_APPSEC_ENABLED: ::String TAG_APM_ENABLED: ::String TAG_DISTRIBUTED_APPSEC_EVENT: ::String + TAG_STACK_TRACE: ::String end end end diff --git a/sig/datadog/tracing/trace_segment.rbs b/sig/datadog/tracing/trace_segment.rbs index 41f81d3114f..656bf2d1d85 100644 --- a/sig/datadog/tracing/trace_segment.rbs +++ b/sig/datadog/tracing/trace_segment.rbs @@ -22,7 +22,7 @@ module Datadog attr_reader sampling_priority: untyped attr_reader service: untyped - def initialize: (untyped spans, ?agent_sample_rate: untyped?, ?hostname: untyped?, ?id: untyped?, ?lang: untyped?, ?name: untyped?, ?origin: untyped?, ?process_id: untyped?, ?rate_limiter_rate: untyped?, ?resource: untyped?, ?root_span_id: untyped?, ?rule_sample_rate: untyped?, ?runtime_id: untyped?, ?sample_rate: untyped?, ?sampling_priority: untyped?, ?service: untyped?, ?tags: untyped?, ?metrics: untyped?) -> void + def initialize: (untyped spans, ?agent_sample_rate: untyped?, ?hostname: untyped?, ?id: untyped?, ?lang: untyped?, ?name: untyped?, ?origin: untyped?, ?process_id: untyped?, ?rate_limiter_rate: untyped?, ?resource: untyped?, ?root_span_id: untyped?, ?rule_sample_rate: untyped?, ?runtime_id: untyped?, ?sample_rate: untyped?, ?sampling_priority: untyped?, ?service: untyped?, ?tags: untyped?, ?metrics: untyped?, ?meta_struct: untyped?) -> void def any?: () -> untyped def count: () -> untyped def empty?: () -> untyped @@ -36,6 +36,7 @@ module Datadog attr_reader root_span_id: untyped attr_reader meta: untyped attr_reader metrics: untyped + attr_reader meta_struct: untyped private