Merge pull request #863 from appsignal/delta-metrics

Store delta gauge of allocated objects in Ruby

.changesets/report-gauge-delta-value-for-allocated-objects.md

@@ -0,0 +1,6 @@
+---
+bump: "patch"
+type: "change"
+---
+
+Report gauge delta value for allocated objects. This reports a more user friendly metric we can graph with a more stable continuous value in apps with stable memory allocation.

lib/appsignal/probes.rb

@@ -3,5 +3,6 @@ module Probes
   end
 end
 
+require "appsignal/probes/helpers"
 require "appsignal/probes/mri"
 require "appsignal/probes/sidekiq"

lib/appsignal/probes/helpers.rb

@@ -0,0 +1,29 @@
+module Appsignal
+  module Probes
+    module Helpers
+      private
+
+      def gauge_delta_cache
+        @gauge_delta_cache ||= {}
+      end
+
+      # Calculate the delta of two values for a gauge metric
+      #
+      # First call will store the data for the metric in the cache and the
+      # second call will return the delta of the gauge metric. This is used for
+      # absolute counter values which we want to track as gauges.
+      #
+      # @example
+      #   gauge_delta :my_cache_key, 10
+      #   gauge_delta :my_cache_key, 15
+      #   # Returns a value of `5`
+      def gauge_delta(cache_key, value)
+        previous_value = gauge_delta_cache[cache_key]
+        gauge_delta_cache[cache_key] = value
+        return unless previous_value
+
+        value - previous_value
+      end
+    end
+  end
+end

lib/appsignal/probes/mri.rb

@@ -1,6 +1,8 @@
 module Appsignal
   module Probes
     class MriProbe
+      include Helpers
+
       # @api private
       def self.dependencies_present?
         defined?(::RubyVM) && ::RubyVM.respond_to?(:stat)
@@ -35,7 +37,14 @@ def call
         @appsignal.set_gauge("gc_total_time", MriProbe.garbage_collection_profiler.total_time)
 
         gc_stats = GC.stat
-        @appsignal.set_gauge("total_allocated_objects", gc_stats[:total_allocated_objects] || gc_stats[:total_allocated_object])
+        allocated_objects =
+          gauge_delta(
+            :allocated_objects,
+            gc_stats[:total_allocated_objects] || gc_stats[:total_allocated_object]
+          )
+        if allocated_objects
+          @appsignal.set_gauge("allocated_objects", allocated_objects)
+        end
 
         @appsignal.add_distribution_value("gc_count", GC.count, :metric => :gc_count)
         @appsignal.add_distribution_value("gc_count", gc_stats[:minor_gc_count], :metric => :minor_gc_count)

lib/appsignal/probes/sidekiq.rb

@@ -1,6 +1,8 @@
 module Appsignal
   module Probes
     class SidekiqProbe
+      include Helpers
+
       # @api private
       attr_reader :config
 
@@ -42,11 +44,15 @@ def track_stats
 
         gauge "worker_count", stats.workers_size
         gauge "process_count", stats.processes_size
-        gauge_delta :jobs_processed, "job_count", stats.processed,
-          :status => :processed
-        gauge_delta :jobs_failed, "job_count", stats.failed, :status => :failed
+        jobs_processed = gauge_delta :jobs_processed, stats.processed
+        if jobs_processed
+          gauge "job_count", jobs_processed, :status => :processed
+        end
+        jobs_failed = gauge_delta :jobs_failed, stats.failed
+        gauge "job_count", jobs_failed, :status => :failed if jobs_failed
         gauge "job_count", stats.retry_size, :status => :retry_queue
-        gauge_delta :jobs_dead, "job_count", stats.dead_size, :status => :died
+        jobs_dead = gauge_delta :jobs_dead, stats.dead_size
+        gauge "job_count", jobs_dead, :status => :died if jobs_dead
         gauge "job_count", stats.scheduled_size, :status => :scheduled
         gauge "job_count", stats.enqueued, :status => :enqueued
       end
@@ -65,25 +71,6 @@ def gauge(key, value, tags = {})
         Appsignal.set_gauge "sidekiq_#{key}", value, tags
       end
 
-      # Track the delta of two values for a gauge metric
-      #
-      # First call will store the data for the metric and the second call will
-      # set a gauge metric with the difference. This is used for absolute
-      # counter values which we want to track as gauges.
-      #
-      # @example
-      #   gauge_delta :my_cache_key, "my_gauge", 10
-      #   gauge_delta :my_cache_key, "my_gauge", 15
-      #   # Creates a gauge with the value `5`
-      # @see #gauge
-      def gauge_delta(cache_key, key, value, tags = {})
-        previous_value = cache[cache_key]
-        cache[cache_key] = value
-        return unless previous_value
-        new_value = value - previous_value
-        gauge key, new_value, tags
-      end
-
       def hostname
         return @hostname if defined?(@hostname)
         if config.key?(:hostname)

spec/lib/appsignal/probes/mri_spec.rb

@@ -33,31 +33,42 @@ def set_gauge(*args) # rubocop:disable Naming/AccessorMethodName
 
   unless DependencyHelper.running_jruby? || DependencyHelper.running_ruby_2_0?
     describe "#call" do
-      before do
-        probe.call
-      end
-
       it "should track vm metrics" do
+        probe.call
         expect_distribution_value("ruby_vm", :class_serial)
         expect_distribution_value("ruby_vm", :global_constant_state)
       end
 
       it "tracks thread counts" do
+        probe.call
         expect_gauge_value("thread_count")
       end
 
       it "tracks GC total time" do
+        probe.call
         expect_gauge_value("gc_total_time")
       end
 
       it "tracks GC runs" do
+        probe.call
         expect_distribution_value("gc_count", :gc_count)
         expect_distribution_value("gc_count", :major_gc_count)
         expect_distribution_value("gc_count", :minor_gc_count)
       end
 
-      it "tracks GC stats" do
-        expect_gauge_value("total_allocated_objects")
+      it "tracks object allocation" do
+        expect(GC).to receive(:stat).and_return(
+          { :total_allocated_objects => 10 },
+          :total_allocated_objects => 15
+        )
+        # Only tracks delta value so the needs to be called twice
+        probe.call
+        probe.call
+        expect_gauge_value("allocated_objects", 5)
+      end
+
+      it "tracks heap slots" do
+        probe.call
         expect_distribution_value("heap_slots", :heap_live)
         expect_distribution_value("heap_slots", :heap_free)
       end
@@ -73,10 +84,10 @@ def expect_distribution_value(expected_key, metric)
     end
   end
 
-  def expect_gauge_value(key)
+  def expect_gauge_value(expected_key, expected_value = nil)
     expect(appsignal_mock.gauges).to satisfy do |gauges|
-      gauges.any? do |gauge|
-        gauge.first == key && !gauge.last.nil?
+      gauges.any? do |(key, value)|
+        expected_key == key && expected_value ? expected_value == value : !value.nil?
       end
     end
   end