[WIP] Component dependency manager

lib/datadog/appsec/component.rb

@@ -8,31 +8,40 @@ module Datadog
   module AppSec
     # Core-pluggable component for AppSec
     class Component
-      class << self
-        def build_appsec_component(settings)
-          return unless settings.respond_to?(:appsec) && settings.appsec.enabled
+      class AppSec
+        extend Core::Dependency
 
-          processor = create_processor(settings)
-          new(processor: processor)
+        setting(:enabled, 'appsec.enabled')
+        setting(:ruleset, 'appsec.ruleset')
+        setting(:ip_denylist, 'appsec.ip_denylist')
+        setting(:user_id_denylist, 'appsec.user_id_denylist')
+        def self.new(enabled, ruleset, ip_denylist, user_id_denylist)
+          return unless enabled
+
+          processor = create_processor(ruleset, ip_denylist, user_id_denylist)
+          Datadog::AppSec::Component.new(processor: processor)
         end
+        # def build_appsec_component(settings)
+        #   return unless settings.respond_to?(:appsec) && settings.appsec.enabled
+        #
+        #   processor = create_processor(settings)
+        #   new(processor: processor)
+        # end
 
         private
 
-        def create_processor(settings)
-          rules = AppSec::Processor::RuleLoader.load_rules(ruleset: settings.appsec.ruleset)
+        def self.create_processor(ruleset, ip_denylist, user_id_denylist)
+          rules = Datadog::AppSec::Processor::RuleLoader.load_rules(ruleset: ruleset)
           return nil unless rules
 
-          data = AppSec::Processor::RuleLoader.load_data(
-            ip_denylist: settings.appsec.ip_denylist,
-            user_id_denylist: settings.appsec.user_id_denylist
-          )
+          data = Datadog::AppSec::Processor::RuleLoader.load_data(ip_denylist: ip_denylist, user_id_denylist: user_id_denylist)
 
-          ruleset = AppSec::Processor::RuleMerger.merge(
+          ruleset = Datadog::AppSec::Processor::RuleMerger.merge(
             rules: [rules],
             data: data,
           )
 
-          processor = Processor.new(ruleset: ruleset)
+          processor = Datadog::AppSec::Processor.new(ruleset: ruleset)
           return nil unless processor.ready?
 
           processor

lib/datadog/ci/configuration/components.rb

@@ -25,6 +25,8 @@ def activate_ci!(settings)
 
           # Pass through any other options
           settings.tracing.test_mode.writer_options = settings.ci.writer_options
+
+          Datadog::Core.dependency_registry.configuration = settings
         end
       end
     end

lib/datadog/core.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'dependency_registry'
 require_relative 'core/extensions'
 
 # We must load core extensions to make certain global APIs

lib/datadog/core/configuration.rb

@@ -1,5 +1,6 @@
 require_relative 'configuration/components'
 require_relative 'configuration/settings'
+require_relative 'dependency'
 require_relative 'telemetry/emitter'
 require_relative 'logger'
 require_relative 'pin'
@@ -182,10 +183,20 @@ def logger
       # Components won't be automatically reinitialized after a shutdown.
       def shutdown!
         safely_synchronize do
-          @components.shutdown! if components?
+          if components?
+            # @components.shutdown!
+            Datadog::Core.dependency_registry.shutdown
+          end
         end
       end
 
+      # TODO: created based on global, declarative registry.
+      # TODO: this one is instantiated, and be reset.
+      # def dependencies
+        # Dummy implementation until DSL declaration registry is separated from live dependency resolution.
+        # Core.dependency_registry
+      # end
+
       protected
 
       def components(allow_initialization: true)
@@ -215,6 +226,9 @@ def reset!
 
           write_components.call(nil)
           configuration.reset!
+          Datadog::AppSec.settings.send(:reset!)
+
+          Datadog::Core.dependency_registry.shutdown
         end
       end
 
@@ -245,12 +259,19 @@ def components?
       end
 
       def build_components(settings)
+        Datadog::Core.dependency_registry.change_settings({}, force_reset_all: true)
+
         components = Components.new(settings)
         components.startup!(settings)
         components
       end
 
       def replace_components!(settings, old)
+        # We don't track changed settings today on `Datadog.configure`, so we
+        # reconfigure all registry components instead.
+        # TODO: keep track of changed settings to only reconfigure changed registry components
+        Datadog::Core.dependency_registry.change_settings({}, force_reset_all: true)
+
         components = Components.new(settings)
 
         old.shutdown!(components)

lib/datadog/core/configuration/agent_settings_resolver.rb

@@ -17,6 +17,23 @@ module Configuration
       # Whenever there is a conflict (different configurations are provided in different orders), it MUST warn the users
       # about it and pick a value based on the following priority: code > environment variable > defaults.
       class AgentSettingsResolver
+        extend Core::Dependency
+
+        component_name('agent_settings')
+        setting(:host, 'agent.host')
+        setting(:port, 'agent.port')
+        setting(:transport_options, 'tracing.transport_options')
+        component(:logger)
+        def self.new(
+          host,
+          port,
+          transport_options,
+          logger:
+        )
+          new = super
+          new.send(:call)
+        end
+
         AgentSettings = \
           Struct.new(
             :adapter,
@@ -49,8 +66,14 @@ def initialize(
             end
           end
 
+        # Used for testing only.
+        # TODO: should we delete/hide this, given tests are doing something prod is never doing?
         def self.call(settings, logger: Datadog.logger)
-          new(settings, logger: logger).send(:call)
+          raise "you goofed, don't call this from prod" unless caller[1].include?('spec/')
+
+          new(settings.agent.host,
+          settings.agent.port,
+          settings.tracing.transport_options, logger: logger)
         end
 
         private
@@ -59,8 +82,10 @@ def self.call(settings, logger: Datadog.logger)
           :logger,
           :settings
 
-        def initialize(settings, logger: Datadog.logger)
-          @settings = settings
+        def initialize(host, port, transport_options, logger: Datadog.logger)
+          @host = host
+          @port = port
+          @transport_options_setting = transport_options
           @logger = logger
         end
 
@@ -112,7 +137,7 @@ def configured_hostname
             ),
             DetectedConfiguration.new(
               friendly_name: "'c.agent.host'",
-              value: settings.agent.host
+              value: @host
             ),
             DetectedConfiguration.new(
               friendly_name: "#{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL} environment variable",
@@ -135,7 +160,7 @@ def configured_port
             ),
             try_parsing_as_integer(
               friendly_name: '"c.agent.port"',
-              value: settings.agent.port,
+              value: @port,
             ),
             DetectedConfiguration.new(
               friendly_name: "#{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL} environment variable",
@@ -201,8 +226,8 @@ def timeout_seconds
         # In transport_options, we try to invoke the transport_options proc and get its configuration. In case that
         # doesn't work, we include the proc directly in the agent settings result.
         def deprecated_for_removal_transport_configuration_proc
-          if settings.tracing.transport_options.is_a?(Proc) && transport_options.adapter.nil?
-            settings.tracing.transport_options
+          if @transport_options_setting.is_a?(Proc) && transport_options.adapter.nil?
+            @transport_options_setting
           end
         end
 
@@ -324,7 +349,7 @@ def mixed_http_and_uds?
         def transport_options
           return @transport_options if defined?(@transport_options)
 
-          transport_options_proc = settings.tracing.transport_options
+          transport_options_proc = @transport_options_setting
 
           @transport_options = TransportOptions.new
 

lib/datadog/core/configuration/components.rb

@@ -19,47 +19,68 @@ class Components
         class << self
           include Datadog::Tracing::Component
 
-          def build_health_metrics(settings)
-            settings = settings.diagnostics.health_metrics
-            options = { enabled: settings.enabled }
-            options[:statsd] = settings.statsd unless settings.statsd.nil?
-
-            Core::Diagnostics::Health::Metrics.new(**options)
+          def build_health_metrics
+            # settings = settings.diagnostics.health_metrics
+            # options = { enabled: settings.enabled }
+            # options[:statsd] = settings.statsd unless settings.statsd.nil?
+            #
+            # Core::Diagnostics::Health::Metrics.new(**options)
+
+            Datadog::Core.dependency_registry.resolve_component(:health_metrics)
           end
 
           def build_logger(settings)
-            logger = settings.logger.instance || Core::Logger.new($stdout)
-            logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
+            # logger = settings.logger.instance || Core::Logger.new($stdout)
+            # logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
+            #
+            # logger
 
-            logger
+            Datadog::Core.dependency_registry.resolve_component(:logger)
           end
 
-          def build_runtime_metrics(settings)
-            options = { enabled: settings.runtime_metrics.enabled }
-            options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
-            options[:services] = [settings.service] unless settings.service.nil?
+          class Logger
+            extend Core::Dependency
+
+            setting(:instance, 'logger.instance')
+            setting(:level, 'logger.level')
+            setting(:debug, 'diagnostics.debug')
+            def self.new(instance, level, debug)
+              logger = instance || Core::Logger.new($stdout)
+              logger.level = debug ? ::Logger::DEBUG : level
 
-            Core::Runtime::Metrics.new(**options)
+              logger
+            end
           end
 
-          def build_runtime_metrics_worker(settings)
-            # NOTE: Should we just ignore building the worker if its not enabled?
-            options = settings.runtime_metrics.opts.merge(
-              enabled: settings.runtime_metrics.enabled,
-              metrics: build_runtime_metrics(settings)
-            )
+          # def build_runtime_metrics(settings)
+          #   options = { enabled: settings.runtime_metrics.enabled }
+          #   options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
+          #   options[:services] = [settings.service] unless settings.service.nil?
+          #
+          #   Core::Runtime::Metrics.new(**options)
+          # end
 
-            Core::Workers::RuntimeMetrics.new(options)
+          def build_runtime_metrics_worker
+            # NOTE: Should we just ignore building the worker if its not enabled?
+            # options = settings.runtime_metrics.opts.merge(
+            #   enabled: settings.runtime_metrics.enabled,
+            #   metrics: build_runtime_metrics(settings)
+            # )
+            #
+            # Core::Workers::RuntimeMetrics.new(**options)
+
+            Datadog::Core.dependency_registry.resolve_component(:runtime_metrics)
           end
 
-          def build_telemetry(settings, agent_settings, logger)
-            enabled = settings.telemetry.enabled
-            if agent_settings.adapter != Datadog::Transport::Ext::HTTP::ADAPTER
-              enabled = false
-              logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
-            end
+          def build_telemetry
+            # enabled = settings.telemetry.enabled
+            # if agent_settings.adapter != Datadog::Transport::Ext::HTTP::ADAPTER
+            #   enabled = false
+            #   logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
+            # end
 
-            Telemetry::Client.new(enabled: enabled)
+            # Telemetry::Client.new # (enabled: enabled)
+            Datadog::Core.dependency_registry.resolve_component(:telemetry)
           end
         end
 
@@ -74,21 +95,15 @@ def build_telemetry(settings, agent_settings, logger)
           :appsec
 
         def initialize(settings)
-          @logger = self.class.build_logger(settings)
-
-          agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
-
-          @remote = Remote::Component.build(settings, agent_settings)
-          @tracer = self.class.build_tracer(settings, agent_settings)
-          @profiler = Datadog::Profiling::Component.build_profiler_component(
-            settings: settings,
-            agent_settings: agent_settings,
-            optional_tracer: @tracer,
-          )
-          @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
-          @health_metrics = self.class.build_health_metrics(settings)
-          @telemetry = self.class.build_telemetry(settings, agent_settings, logger)
-          @appsec = Datadog::AppSec::Component.build_appsec_component(settings)
+          @logger = Datadog::Core.dependency_registry.resolve_component(:logger)
+
+          @remote = Datadog::Core.dependency_registry.resolve_component(:remote)
+          @tracer = Datadog::Core.dependency_registry.resolve_component(:tracer)
+          @profiler = Datadog::Core.dependency_registry.resolve_component(:profiler)
+          @runtime_metrics = Datadog::Core.dependency_registry.resolve_component(:runtime_metrics)
+          @health_metrics = Datadog::Core.dependency_registry.resolve_component(:health_metrics)
+          @telemetry = Datadog::Core.dependency_registry.resolve_component(:telemetry)
+          @appsec = Datadog::Core.dependency_registry.resolve_component(:app_sec)
         end
 
         # Starts up components