Merge pull request #4334 from DataDog/vpellan/meta-struct

Add meta_struct support on traces and spans

Author: y9v

Steepfile

@@ -160,6 +160,7 @@ target :datadog do
   library 'zlib'
   library 'time'
   library 'pp'
+  library 'forwardable'
 
   # Load all dependency signatures from the `vendor/rbs` directory
   repo_path 'vendor/rbs'

lib/datadog/tracing/metadata.rb

@@ -2,6 +2,7 @@
 
 require_relative 'metadata/analytics'
 require_relative 'metadata/tagging'
+require_relative 'metadata/metastruct_tagging'
 require_relative 'metadata/errors'
 
 module Datadog
@@ -10,6 +11,7 @@ module Tracing
     module Metadata
       def self.included(base)
         base.include(Metadata::Tagging)
+        base.include(Metadata::MetastructTagging)
         base.include(Metadata::Errors)
 
         # Additional extensions

lib/datadog/tracing/metadata/metastruct.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Datadog
+  module Tracing
+    module Metadata
+      # This class is a data structure that is used to store
+      # complex metadata, such as an array of objects.
+      #
+      # It is serialized to MessagePack format when sent to the agent.
+      class Metastruct
+        extend Forwardable
+
+        def_delegators :@metastruct, :[], :[]=, :to_h
+
+        def initialize
+          @metastruct = {}
+        end
+
+        def to_msgpack(packer = nil)
+          # JRuby doesn't pass the packer
+          packer ||= MessagePack::Packer.new
+
+          packer.write(@metastruct.transform_values(&:to_msgpack))
+        end
+
+        def pretty_print(q)
+          q.seplist @metastruct.each do |key, value|
+            q.text "#{key} => #{value}"
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/tracing/metadata/metastruct_tagging.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative 'metastruct'
+
+module Datadog
+  module Tracing
+    module Metadata
+      # Adds data storage for the `meta_struct` field.
+      #
+      # This field is used to send more complex data like an array of objects
+      # in MessagePack format to the agent, and has no size limitations.
+      #
+      # The agent fully supports meta_struct from version v7.35.0 (April 2022).
+      #
+      # On versions older than v7.35.0, sending traces containing meta_struct
+      # has no unexpected side-effects; traces are sent to the backend as expected,
+      # while the meta_struct field is stripped.
+      module MetastructTagging
+        # Set the given key / value tag pair on the metastruct.
+        #
+        # A valid example is:
+        #
+        #   span.set_metastruct_tag('_dd.stack', [])
+        def set_metastruct_tag(key, value)
+          metastruct[key] = value
+        end
+
+        # Return the metastruct tag value for the given key,
+        # returns nil if the key doesn't exist.
+        def get_metastruct_tag(key)
+          metastruct[key]
+        end
+
+        private
+
+        def metastruct
+          @metastruct ||= Metastruct.new
+        end
+      end
+    end
+  end
+end

lib/datadog/tracing/span.rb

@@ -33,6 +33,9 @@ class Span
         :status,
         :trace_id
 
+      attr_reader \
+        :metastruct
+
       attr_writer \
         :duration
 
@@ -54,6 +57,7 @@ def initialize(
         id: nil,
         meta: nil,
         metrics: nil,
+        metastruct: nil,
         parent_id: 0,
         resource: name,
         service: nil,
@@ -76,6 +80,7 @@ def initialize(
 
         @meta = meta || {}
         @metrics = metrics || {}
+        @metastruct = metastruct || {}
         @status = status || 0
 
         # start_time and end_time track wall clock. In Ruby, wall clock
@@ -144,6 +149,7 @@ def to_hash
           error: @status,
           meta: @meta,
           metrics: @metrics,
+          meta_struct: @metastruct.to_h,
           name: @name,
           parent_id: @parent_id,
           resource: @resource,
@@ -185,12 +191,15 @@ def pretty_print(q)
               q.text "#{key} => #{value}"
             end
           end
-          q.group(2, 'Metrics: [', ']') do
+          q.group(2, 'Metrics: [', "]\n") do
             q.breakable
             q.seplist @metrics.each do |key, value|
               q.text "#{key} => #{value}"
             end
           end
+          q.group(2, 'Metastruct: [', ']') do
+            metastruct.pretty_print(q)
+          end
         end
       end
 

lib/datadog/tracing/span_operation.rb

@@ -289,6 +289,7 @@ def to_hash
           id: @id,
           meta: meta,
           metrics: metrics,
+          metastruct: metastruct,
           name: @name,
           parent_id: @parent_id,
           resource: @resource,
@@ -328,12 +329,15 @@ def pretty_print(q)
               q.text "#{key} => #{value}"
             end
           end
-          q.group(2, 'Metrics: [', ']') do
+          q.group(2, 'Metrics: [', "]\n") do
             q.breakable
             q.seplist metrics.each do |key, value|
               q.text "#{key} => #{value}"
             end
           end
+          q.group(2, 'Metastruct: [', ']') do
+            metastruct.pretty_print(q)
+          end
         end
       end
 
@@ -456,6 +460,7 @@ def build_span
           id: @id,
           meta: Core::Utils::SafeDup.frozen_or_dup(meta),
           metrics: Core::Utils::SafeDup.frozen_or_dup(metrics),
+          metastruct: Core::Utils::SafeDup.frozen_or_dup(metastruct),
           parent_id: @parent_id,
           resource: @resource,
           service: @service,

lib/datadog/tracing/transport/serializable_trace.rb

@@ -69,7 +69,7 @@ def initialize(span, native_events_supported:)
         def to_msgpack(packer = nil)
           packer ||= MessagePack::Packer.new
 
-          number_of_elements_to_write = 11
+          number_of_elements_to_write = 12
 
           number_of_elements_to_write += 1 if span.events.any? && @native_events_supported
 
@@ -117,6 +117,8 @@ def to_msgpack(packer = nil)
           packer.write(span.meta)
           packer.write('metrics')
           packer.write(span.metrics)
+          packer.write('meta_struct')
+          packer.write(span.metastruct)
           packer.write('span_links')
           packer.write(span.links.map(&:to_hash))
           packer.write('error')

sig/datadog/tracing/metadata.rbs

@@ -2,6 +2,7 @@ module Datadog
   module Tracing
     module Metadata
       include Metadata::Tagging
+      include Metadata::MetastructTagging
       include Metadata::Errors
       prepend Metadata::Analytics
     end

sig/datadog/tracing/metadata/metastruct.rbs

@@ -0,0 +1,19 @@
+module Datadog
+  module Tracing
+    module Metadata
+      class Metastruct
+        extend Forwardable
+
+        def []: (String | Symbol key) -> untyped
+
+        def []=: (String | Symbol key, untyped value) -> untyped
+
+        def to_h: () -> ::Hash[String | Symbol, untyped]
+
+        def to_msgpack: (untyped packer) -> void
+
+        def pretty_print: (untyped q) -> void
+      end
+    end
+  end
+end

sig/datadog/tracing/metadata/metastruct_tagging.rbs

@@ -0,0 +1,11 @@
+module Datadog
+  module Tracing
+    module Metadata
+      module MetastructTagging
+        def set_metastruct_tag: (String | Symbol key, untyped value) -> untyped
+        def get_metastruct_tag: (String | Symbol key) -> untyped
+        private def metastruct: () -> Datadog::Tracing::Metadata::Metastruct
+      end
+    end
+  end
+end

sig/datadog/tracing/span.rbs

@@ -18,6 +18,7 @@ module Datadog
       attr_accessor status: Integer
       attr_accessor trace_id: Integer
       attr_writer duration: (Float | nil)
+      attr_reader metastruct: Hash[String | Symbol, untyped]
 
       def initialize: (
           String name,
@@ -26,6 +27,7 @@ module Datadog
           ?id: (Integer | nil),
           ?meta: (Hash[String, String] | nil),
           ?metrics: (Hash[String, Float] | nil),
+          ?metastruct: Hash[String | Symbol, untyped],
           ?parent_id: Integer,
           ?resource: String,
           ?service: (String | nil),

spec/datadog/tracing/metadata/metastruct_tagging_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+require 'datadog/tracing/metadata/metastruct_tagging'
+
+RSpec.describe Datadog::Tracing::Metadata::MetastructTagging do
+  subject(:object) { test_class.new }
+
+  let(:test_class) do
+    Class.new { include Datadog::Tracing::Metadata::MetastructTagging }
+  end
+
+  describe '#set_metastruct_tag' do
+    it 'sets the metastruct to a hash with given key / value pair' do
+      expect { object.set_metastruct_tag(:foo, [{ some: 'value' }]) }
+        .to change { object.get_metastruct_tag(:foo) }.from(nil).to([{ some: 'value' }])
+    end
+
+    it 'does not lose previous entries' do
+      object.set_metastruct_tag(:foo, [{ some: 'value' }])
+
+      expect { object.set_metastruct_tag(:bar, [{ another: 'value' }]) }
+        .not_to(change { object.get_metastruct_tag(:foo) })
+    end
+  end
+
+  describe '#get_metastruct_tag' do
+    it 'returns nil if the key does not exist' do
+      expect(object.get_metastruct_tag(:foo)).to be_nil
+    end
+  end
+end

spec/datadog/tracing/span_spec.rb

@@ -244,6 +244,7 @@
         type: nil,
         meta: {},
         metrics: {},
+        meta_struct: {},
         span_links: [],
         error: 0
       )

spec/datadog/tracing/transport/serializable_trace_spec.rb

@@ -40,6 +40,7 @@
           'type',
           'meta',
           'metrics',
+          'meta_struct',
           'span_links',
           'error',
         ]