Merge pull request #1433 from fluent/move-remove-keys-to-last

record_transformer: remove_keys processing should be last
fluent · Feb 1, 2017 · c66ef65 · c66ef65
1 parent f57a6aa
commit c66ef65
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 5 deletions.
diff --git a/lib/fluent/plugin/filter_record_transformer.rb b/lib/fluent/plugin/filter_record_transformer.rb
@@ -95,14 +95,13 @@ def filter_stream(tag, es)
       last_record = nil
       es.each do |time, record|
         last_record = record # for debug log
-        placeholder_values.merge!({
-          'time'     => @placeholder_expander.time_value(time),
-          'record'   => record,
-        })
+        placeholder_values['time'] = @placeholder_expander.time_value(time)
+        placeholder_values['record'] = record
         new_record = reform(record, placeholder_values)
         if @renew_time_key && new_record.has_key?(@renew_time_key)
           time = new_record[@renew_time_key].to_i
         end
+        @remove_keys.each { |k| new_record.delete(k) } if @remove_keys
         new_es.add(time, new_record)
       end
       new_es
@@ -131,7 +130,6 @@ def reform(record, placeholder_values)
       new_record = @renew_record ? {} : record.dup
       @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
       new_record.merge!(expand_placeholders(@map, placeholders))
-      @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
 
       new_record
     end

diff --git a/test/plugin/test_filter_record_transformer.rb b/test/plugin/test_filter_record_transformer.rb
@@ -99,6 +99,25 @@ def emit(config, msgs = [''])
       es = emit(config, msgs)
       es.each_with_index do |(time, record), i|
         assert_equal(times[i].to_i, time)
+        assert_true(record.has_key?('message'))
+      end
+    end
+
+    test 'renew_time_key and remove_keys' do
+      config = %[
+                 renew_time_key event_time_key
+                 remove_keys event_time_key
+                 auto_typecast true
+                 <record>
+                   event_time_key ${record["message"]}
+                 </record>
+               ]
+      times = [Time.local(2, 2, 3, 4, 5, 2010, nil, nil, nil, nil), Time.local(3, 2, 3, 4, 5, 2010, nil, nil, nil, nil)]
+      msgs = times.map { |t| t.to_f.to_s }
+      filtered = emit(config, msgs)
+      filtered.each_with_index do |(time, _record), i|
+        assert_equal(times[i].to_i, time)
+        assert_false(_record.has_key?('event_time_key'))
       end
     end